From 869ceb0398e2d3357260f4458e9aaccae01798fc Mon Sep 17 00:00:00 2001 From: Matthias Rampke Date: Tue, 26 Jun 2012 20:04:11 +0200 Subject: [PATCH 1/7] use /bin/sh as bash is not available everywhere the script didn't use bash features anyway (at least as far as I noticed) --- fetch-tweets.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fetch-tweets.sh b/fetch-tweets.sh index 765b353..8a0331b 100755 --- a/fetch-tweets.sh +++ b/fetch-tweets.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/bin/sh # vim: set ts=4 sw=4 umask 002 From d3443a2a8ac5ca99fff8b77b0fbd4f3812c6f5f1 Mon Sep 17 00:00:00 2001 From: Matthias Rampke Date: Tue, 26 Jun 2012 20:04:50 +0200 Subject: [PATCH 2/7] specify template to mktemp(1) mktemp(1) is not specified by POSIX and accordingly inconsistent across systems - e.g. non-GNU-implementations requiring that there be some sort of template specified. this appears to be the simplest more-or-less portable invocation. use the user name as part of the name for temporary files. --- fetch-tweets.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fetch-tweets.sh b/fetch-tweets.sh index 8a0331b..8b5710e 100755 --- a/fetch-tweets.sh +++ b/fetch-tweets.sh @@ -36,8 +36,8 @@ do echo $1 tweet total "$twitter_total" is greater than the already saved "$saved" echo Trying to get $(($twitter_total - $saved)) -temp=$(mktemp) -temp2=$(mktemp) +temp=$(mktemp "$1.XXXX") +temp2=$(mktemp "$1.XXXX") url="${api}screen_name=${1}&count=200&page=${page}${since}&include_rts=true&trim_user=1&include_entities=1" From 05d1e94366e6e83e0e957ff06c5a88804eae537d Mon Sep 17 00:00:00 2001 From: Matthias Rampke Date: Tue, 26 Jun 2012 20:13:15 +0200 Subject: [PATCH 3/7] find xmlstarlet even if it is called xml which is the case with at least the Homebrew (OS X) and pkgsrc (many) package managers. FIXME: doesn't check if 'xml' is xmlstarlet TODO: fail explicitly if no --- fetch-tweets.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/fetch-tweets.sh b/fetch-tweets.sh index 8b5710e..dfa9c25 100755 --- a/fetch-tweets.sh +++ b/fetch-tweets.sh @@ -10,6 +10,11 @@ then exit 1 fi +# xmlstarlet is often just xml +if ! which xmlstarlet >> /dev/null && which xml >> /dev/null; then + alias xmlstarlet=xml +fi + twitter_total=$(curl -s "http://api.twitter.com/1/users/lookup.xml?screen_name=$1" | xmlstarlet sel -t -m "//users/user/statuses_count" -v .) From 14b68472057581150ed0aaa7c4fc8b539ebd61f0 Mon Sep 17 00:00:00 2001 From: Matthias Rampke Date: Tue, 26 Jun 2012 20:44:03 +0200 Subject: [PATCH 4/7] use printf instead of echo -e echo is not very portable, printf is well-specified (fail case: dash's builtin echo doesn't know -e) --- fetch-tweets.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fetch-tweets.sh b/fetch-tweets.sh index dfa9c25..499707c 100755 --- a/fetch-tweets.sh +++ b/fetch-tweets.sh @@ -6,7 +6,7 @@ api="http://api.twitter.com/1/statuses/user_timeline.xml?" if ! test "$1" then - echo -e "Please specify twitter username\n e.g. $0 kaihendry" + printf "Please specify twitter username\n e.g. %s kaihendry\n" $0 exit 1 fi From 94e310791dc509ac09000c287735996f165c6d38 Mon Sep 17 00:00:00 2001 From: Matthias Rampke Date: Tue, 26 Jun 2012 21:20:28 +0200 Subject: [PATCH 5/7] re-implement header/data splitting in ed(1) somehow the original way (stdout/stderr) didn't work in dash (which is /bin/sh in Debian, and rather limited, that is, a good testing ground for POSIX shell compatibility) --- fetch-tweets.sh | 31 ++++++++++++++++--------------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/fetch-tweets.sh b/fetch-tweets.sh index 499707c..6a14a0c 100755 --- a/fetch-tweets.sh +++ b/fetch-tweets.sh @@ -47,24 +47,25 @@ temp2=$(mktemp "$1.XXXX") url="${api}screen_name=${1}&count=200&page=${page}${since}&include_rts=true&trim_user=1&include_entities=1" echo "curl -s \"$url\"" -curl -si "$url" > $temp +curl -si "$url" | tee $temp2 > $temp echo $? -{ -{ while read -r -do -if test "$REPLY" = $'\r' -then - break -else - echo "$REPLY" >&2 # print header to stderr -fi -done -cat; } < $temp > $temp2 -} 2>&1 | # redirect back to stdout for grep -grep -iE 'rate|status' # show the interesting twitter rate limits +# keep only headers in $temp2 +ed -s $temp2 << "EOF_ED1" +/^[[:space:]]*$/ +.,$d +wq +EOF_ED1 -mv $temp2 $temp +# keep only content in $temp +ed -s $temp << "EOF_ED2" +/^[[:space:]]*$/ +1,.d +wq +EOF_ED2 + + +grep -iE 'rate|status' $temp2 # show the interesting twitter rate limits if test "$(xmlstarlet sel -t -v "count(//statuses/status)" $temp 2>/dev/null)" -eq 0 then From ebc6ce12af1c3b1a7fb119549991982c1aad88db Mon Sep 17 00:00:00 2001 From: Matthias Rampke Date: Tue, 26 Jun 2012 21:22:16 +0200 Subject: [PATCH 6/7] remove unportable test syntax [[ ]] is completely unnecessary, [ ] does the trick fine. --- fetch-tweets.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fetch-tweets.sh b/fetch-tweets.sh index 6a14a0c..411413e 100755 --- a/fetch-tweets.sh +++ b/fetch-tweets.sh @@ -96,7 +96,7 @@ do set -- $(echo $rest) finUrl=$2 domain=$(echo $finUrl | cut -d'/' -f3) - if [[ "$shortDomains" = *$domain* ]] + if [ "$shortDomains" = *$domain* ] then finUrl=$(curl "$finUrl" -s -L -I -o /dev/null -w '%{url_effective}') fi From c6f524b5e56f5515ef4257a84c031b040bab5eae Mon Sep 17 00:00:00 2001 From: Matthias Rampke Date: Tue, 26 Jun 2012 21:54:19 +0200 Subject: [PATCH 7/7] use shell variable instead of alias for switching xmlstarlet command. bash doesn't like aliases. --- fetch-tweets.sh | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/fetch-tweets.sh b/fetch-tweets.sh index 411413e..4efd62e 100755 --- a/fetch-tweets.sh +++ b/fetch-tweets.sh @@ -11,12 +11,17 @@ then fi # xmlstarlet is often just xml -if ! which xmlstarlet >> /dev/null && which xml >> /dev/null; then - alias xmlstarlet=xml +if which xmlstarlet >> /dev/null; then + XMLSTARLET=xmlstarled +elif which xml >> /dev/null; then + XMLSTARLET=xml +else + echo "xmlstarlet not found :(" + exit 1 fi twitter_total=$(curl -s "http://api.twitter.com/1/users/lookup.xml?screen_name=$1" | -xmlstarlet sel -t -m "//users/user/statuses_count" -v .) +$XMLSTARLET sel -t -m "//users/user/statuses_count" -v .) if ! test "$twitter_total" -gt 0 2>/dev/null then @@ -67,7 +72,7 @@ EOF_ED2 grep -iE 'rate|status' $temp2 # show the interesting twitter rate limits -if test "$(xmlstarlet sel -t -v "count(//statuses/status)" $temp 2>/dev/null)" -eq 0 +if test "$($XMLSTARLET sel -t -v "count(//statuses/status)" $temp 2>/dev/null)" -eq 0 then head $temp | grep -q "Over capacity" && echo "Twitter is OVER CAPACITY" if test "$2" && test "$since" @@ -85,7 +90,7 @@ fi shortDomains="t.co bit.ly tinyurl.com goo.gl" -xmlstarlet sel -t -m "statuses/status" -n -o "text " -v "id" -o "|" -v "created_at" -o "|" -v "normalize-space(text)" \ +$XMLSTARLET sel -t -m "statuses/status" -n -o "text " -v "id" -o "|" -v "created_at" -o "|" -v "normalize-space(text)" \ -m "entities/urls/url" -i "expanded_url != ''" -n -o "url " -v "url" -o " " -v "expanded_url" $temp | { while read -r first rest