pacnpal/greptweet
1
0
Fork 0
mirror of https://github.com/thewesker/greptweet.git synced 2025-12-20 12:11:05 -05:00
Code Issues Packages Projects Releases Wiki Activity

small code cleanup

Browse Source
This commit is contained in:
Kai Hendry
2012-04-25 21:09:07 +08:00
parent a1c536a967
commit ec3ab7c63f
2 changed files with 15 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
bootstrap

Submodule bootstrap updated: 7f32d15fbb...b0bd3efc4a

30
fetch-tweets.sh
View File

@@ -10,7 +10,8 @@ then
exit 1 exit 1
fi fi
twitter_total=$(curl -s "http://api.twitter.com/1/users/lookup.xml?screen_name=$1" | xmlstarlet sel -t -m "//users/user/statuses_count" -v . 2>/dev/null) twitter_total=$(curl -s "http://api.twitter.com/1/users/lookup.xml?screen_name=$1" |
xmlstarlet sel -t -m "//users/user/statuses_count" -v .)
if ! test "$twitter_total" -gt 0 2>/dev/null if ! test "$twitter_total" -gt 0 2>/dev/null
then then
@@ -20,7 +21,6 @@ fi
page=1 page=1
saved=0 saved=0
stalled=0
if test -s "$1.txt" if test -s "$1.txt"
then then
@@ -81,20 +81,21 @@ shortDomains="t.co bit.ly tinyurl.com goo.gl"
xmlstarlet sel -t -m "statuses/status" -n -o "text " -v "id" -o "|" -v "created_at" -o "|" -v "normalize-space(text)" \ xmlstarlet sel -t -m "statuses/status" -n -o "text " -v "id" -o "|" -v "created_at" -o "|" -v "normalize-space(text)" \
-m "entities/urls/url" -i "expanded_url != ''" -n -o "url " -v "url" -o " " -v "expanded_url" $temp | { -m "entities/urls/url" -i "expanded_url != ''" -n -o "url " -v "url" -o " " -v "expanded_url" $temp | {
while read -r first rest while read -r first rest
do do
case $first in case $first in
"text") echo $text; text=$rest ;; "text") echo $text; text=$rest ;;
"url") "url")
set -- $(echo $rest) set -- $(echo $rest)
finUrl=$2 finUrl=$2
domain=$(echo $finUrl | cut -d'/' -f3) domain=$(echo $finUrl | cut -d'/' -f3)
if [[ "$shortDomains" = *$domain* ]] if [[ "$shortDomains" = *$domain* ]]
then then
finUrl=$(curl "$finUrl" -s -L -I -o /dev/null -w '%{url_effective}') finUrl=$(curl "$finUrl" -s -L -I -o /dev/null -w '%{url_effective}')
fi fi
text=$(echo $text | sed s,$1,$finUrl,g) text=$(echo $text | sed s,$1,$finUrl,g)
;; ;;
esac esac
done done
echo $text echo $text
@@ -110,8 +111,6 @@ then
continue continue
fi fi
#cat $temp2
if test -f $1.txt if test -f $1.txt
then then
mv $1.txt $temp mv $1.txt $temp
@@ -122,6 +121,7 @@ else
fi fi
sort -r -n -u $temp $temp2 > "$1.txt" sort -r -n -u $temp $temp2 > "$1.txt"
rm -f $temp $temp2
after=$(wc -l < "$1.txt") after=$(wc -l < "$1.txt")
echo Before: $before After: $after echo Before: $before After: $after
@@ -129,11 +129,9 @@ echo Before: $before After: $after
if test "$before" -eq "$after" if test "$before" -eq "$after"
then then
echo Unable to retrieve anything new. Approximately $(( $twitter_total - $after)) missing tweets echo Unable to retrieve anything new. Approximately $(( $twitter_total - $after)) missing tweets
rm -f $temp $temp2
exit exit
fi fi
rm -f $temp $temp2
page=$(($page + 1)) page=$(($page + 1))
saved=$(wc -l < "$1.txt") saved=$(wc -l < "$1.txt")
echo $saved echo $saved