pacnpal/greptweet
1
0
Fork 0
mirror of https://github.com/thewesker/greptweet.git synced 2025-12-25 22:31:06 -05:00
Code Issues Packages Projects Releases Wiki Activity

expand urls

Browse Source
This commit is contained in:
JamieKitson
2011-11-13 17:57:14 +00:00
parent 61ae3cecb4
commit 863704da8d
1 changed files with 30 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
fetch-tweets.sh
View File

@@ -37,8 +37,10 @@ echo Trying to get $(($twitter_total - $saved))
temp=$(mktemp)
temp2=$(mktemp)
echo "curl -s \"${api}screen_name=${1}&count=200&page=${page}${since}&include_rts=true&trim_user=1\""
curl -si "${api}screen_name=${1}&count=200&page=${page}${since}&include_rts=true&trim_user=1" > $temp
url="${api}screen_name=${1}&count=200&page=${page}${since}&include_rts=true&trim_user=1&include_entities=1"
echo "curl -s \"$url\""
curl -si "$url" > $temp
echo $?
{
@@ -76,7 +78,17 @@ then
fi
xmlstarlet sel -t -m "//statuses/status" -v "id" -o "|" -v "created_at" -o "|" -v "normalize-space(text)" -n $temp > $temp2
xmlstarlet sel -t -m "statuses/status" -n -o "text " -v "text" -m "entities/urls/url" -i "expanded_url != ''" -n -o "url " -v "url" -o " " -v "expanded_url" $temp | {
while read -r first rest
do
case $first in
"text") echo $text; text=$rest ;;
"url") set -- $(echo $rest); text=$(echo $text | sed s,$1,$2,g) ;;
esac
done
echo $text
} > $temp2
cat $temp2 | perl -MHTML::Entities -pe 'decode_entities($_)' > $temp
cat $temp | sed '/^$/d' > $temp2