pacnpal/greptweet
1
0
Fork 0
mirror of https://github.com/thewesker/greptweet.git synced 2025-12-20 12:11:05 -05:00
Code Issues Packages Projects Releases Wiki Activity

Avoid httpd killing fetchtweets early by executing it off in the background

Browse Source 
TODO: some UI if lock is around, to tell folks a fetch is underway
This commit is contained in:
Kai Hendry
2012-07-24 17:23:31 +02:00
parent 41389fb344
commit 782a06d13f
3 changed files with 16 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files

40
create.cgi
View File

@@ -28,7 +28,7 @@ else
exit exit
fi fi
if test "${parm[2]}" == "o" if test "${parm[2]}" == "o" # unused
then then
test "${parm[3]}" && old=1 test "${parm[3]}" && old=1
fi fi
@@ -45,23 +45,17 @@ cat <<END
<body> <body>
<div class="container"> <div class="container">
<div class="content"> <div class="content">
<h1 class="alert alert-info">Fetching upto 3200 tweets from $id</h1>
<p class="help-inline">Please be patient. Closing this page prematurely can limit the tweets <a href="https://github.com/kaihendry/Greptweet/blob/master/fetch-tweets.sh">fetch-tweets.sh</a> gets and trigger a locking bug.</p>
<pre>
END END
if test -d u/$id if test -d u/$id
then then
echo Attempting an update echo "<h1 class='alert alert-info'>Attempting to update $id</h1>"
else else
if curl -sI http://api.twitter.com/1/users/lookup.xml?screen_name=${id} | if curl -sI http://api.twitter.com/1/users/lookup.xml?screen_name=${id} |
grep -q "Status: 404 Not Found" grep -q "Status: 404 Not Found"
then then
echo "$id does not exist on twitter.com :(" echo "<h1 class='alert alert-info'>$id does not exist on twitter.com :(</h1>"
exit exit
fi fi
@@ -69,27 +63,11 @@ else
fi fi
cd u/$id cd u/$id
ln -sf ../../index.html || true ln -sf ../../index.html || true
ln -sf ../../grep.php || true ln -sf ../../grep.php || true
if ! test -f lock
then
touch lock
../../fetch-tweets.sh $id
rm lock
else
echo Fetching already! Locks are cleared daily
fi
echo "</pre>"
# Clean up in case it went wrong (e.g. trying to retrieve from an account with protected tweets)
if test -s "$oldpwd/u/$id/$id.txt"
then
if echo $id | grep -q -v '_' # Underscores in domain names is a no no if echo $id | grep -q -v '_' # Underscores in domain names is a no no
then then
mkdir /srv/www/$id.greptweet.com 2> /dev/null || true mkdir /srv/www/$id.greptweet.com 2> /dev/null || true
@@ -107,14 +85,6 @@ ln -sf $id.txt tweets.txt
test -h greptweet.appcache && rm -f greptweet.appcache test -h greptweet.appcache && rm -f greptweet.appcache
sed -e "s,TIMESTAMP,$(date)," ../../greptweet.appcache > greptweet.appcache sed -e "s,TIMESTAMP,$(date)," ../../greptweet.appcache > greptweet.appcache
else echo "<pre>"
rm -rf $oldpwd/u/$id
echo '<h1 class="alert alert-error">Sorry the Twitter API is failing. Try again later.</h1>'
fi
cat <<END ../../fetch-tweets.sh $id & disown
</div>
</div>
</body>
</html>
END

2
expand-urls.sh
View File

@@ -5,7 +5,7 @@ test -s "$1" || exit
test "${1##*.}" = 'txt' || exit test "${1##*.}" = 'txt' || exit
temp=$(mktemp "$1.XXXX") temp=$(mktemp "$1.XXXX")
trap "rm -f $temp" INT trap "rm -f $temp" EXIT
IFS='|' IFS='|'
while read -r id date text while read -r id date text

11
fetch-tweets.sh
View File

@@ -1,6 +1,16 @@
#!/usr/bin/env bash #!/usr/bin/env bash
# vim: set ts=4 sw=4 # vim: set ts=4 sw=4
if test -d lock
then
echo "Already running"
exit 1 # trap goes below in order not to remove the lock
else
mkdir lock
fi
trap "rm -vrf $temp $temp2 lock; exit" EXIT
umask 002 umask 002
api="http://api.twitter.com/1/statuses/user_timeline.xml?" api="http://api.twitter.com/1/statuses/user_timeline.xml?"
@@ -42,7 +52,6 @@ echo Trying to get $(($twitter_total - $saved))
temp=$(mktemp "$1.XXXX") temp=$(mktemp "$1.XXXX")
temp2=$(mktemp "$1.XXXX") temp2=$(mktemp "$1.XXXX")
trap "rm -f $temp $temp2; exit" INT EXIT
url="${api}screen_name=${1}&count=200&page=${page}${since}&include_rts=true&trim_user=1&include_entities=1" url="${api}screen_name=${1}&count=200&page=${page}${since}&include_rts=true&trim_user=1&include_entities=1"