diff --git a/expand-urls.sh b/expand-urls.sh index 2972b78..6526c1a 100755 --- a/expand-urls.sh +++ b/expand-urls.sh @@ -1,10 +1,11 @@ #!/usr/bin/env bash # vim: set ts=4 sw=4 -test -s $1 || exit -test ${1##*.} = 'txt' || exit +test -s "$1" || exit +test "${1##*.}" = 'txt' || exit temp=$(mktemp "$1.XXXX") +trap "rm -f $temp" INT IFS='|' while read -r id date text diff --git a/fetch-tweets.sh b/fetch-tweets.sh index 37751d1..cf8f99b 100755 --- a/fetch-tweets.sh +++ b/fetch-tweets.sh @@ -1,6 +1,7 @@ -#!/usr/bin/env bash +#!/bin/sh # vim: set ts=4 sw=4 + umask 002 api="http://api.twitter.com/1/statuses/user_timeline.xml?" @@ -42,6 +43,8 @@ echo Trying to get $(($twitter_total - $saved)) temp=$(mktemp "$1.XXXX") temp2=$(mktemp "$1.XXXX") +trap "rm -f $temp $temp2" INT EXIT + url="${api}screen_name=${1}&count=200&page=${page}${since}&include_rts=true&trim_user=1&include_entities=1" echo "curl -s \"$url\"" @@ -62,7 +65,6 @@ ed -s $temp << "EOF_ED2" wq EOF_ED2 - grep -iE 'rate|status' $temp2 # show the interesting twitter rate limits if test "$(xml sel -t -v "count(//statuses/status)" $temp 2>/dev/null)" -eq 0 @@ -81,22 +83,7 @@ then exit fi -xml sel -t -m "statuses/status" -n -o "text " -v "id" -o "|" -v "created_at" -o "|" -v "normalize-space(text)" \ --m "entities/urls/url" -i "expanded_url != ''" -n -o "url " -v "url" -o " " -v "expanded_url" $temp | { - -while read -r first rest -do - case $first in - "text") echo "$text"; text="$rest" ;; - "url") - set -- $(echo $rest) - expandedURL=$(curl "$2" -m5 -s -L -I -o /dev/null -w '%{url_effective}') - text=${text//"$1"/$expandedURL} # BASHism #11 - ;; - esac -done -echo "$text" -} > $temp2 +xml sel -t -m "statuses/status" -n -v "id" -o "|" -v "created_at" -o "|" -v "normalize-space(text)" $temp > $temp2 perl -MHTML::Entities -pe 'decode_entities($_)' < $temp2 > $temp sed '/^$/d' < $temp > $temp2