pacnpal/iron_ebooks
1
0
Fork 0
mirror of https://github.com/thewesker/iron_ebooks.git synced 2025-12-20 04:11:12 -05:00
Code Issues Packages Projects Releases Wiki Activity

Don't tokenize empty tweets

Browse Source
This commit is contained in:
Jacob Harris
2013-12-28 22:26:21 -05:00
parent 3027681de3
commit 31aa8425de
1 changed files with 1 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
ebook.rb
View File

@@ -82,6 +82,7 @@ else
tokenizer = Punkt::SentenceTokenizer.new(source_tweets.join(" ")) # init with corpus of all sentences tokenizer = Punkt::SentenceTokenizer.new(source_tweets.join(" ")) # init with corpus of all sentences
source_tweets.each do |twt| source_tweets.each do |twt|
next if twt.nil? || twt == ''
sentences = tokenizer.sentences_from_text(twt, :output => :sentences_text) sentences = tokenizer.sentences_from_text(twt, :output => :sentences_text)
# sentences = text.split(/[.:;?!]/) # sentences = text.split(/[.:;?!]/)