Tweaking sentence tokenization, support for contractions

2025-12-24 22:31:16 -05:00 · 2013-08-17 21:38:14 -04:00
parent 7061990d1e
commit b190084a02
5 changed files with 107 additions and 27 deletions
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -0,0 +1,44 @@
+GEM
+  remote: https://rubygems.org/
+  specs:
+    ethon (0.5.12)
+      ffi (>= 1.3.0)
+      mime-types (~> 1.18)
+    faraday (0.8.7)
+      multipart-post (~> 1.1)
+    ffi (1.9.0)
+    iron_core (0.6.2)
+      rest (>= 2.2.0)
+    iron_worker_ng (0.16.4)
+      bundler (>= 1.2.0)
+      iron_core (>= 0.5.1)
+      rubyzip (>= 0.9.9)
+    mime-types (1.23)
+    multi_json (1.7.7)
+    multipart-post (1.2.0)
+    net-http-persistent (2.8)
+    punkt-segmenter (0.9.1)
+      unicode_utils (>= 1.0.0)
+    rest (2.6.3)
+      net-http-persistent
+      rest-client (>= 0.3.0)
+    rest-client (1.6.7)
+      mime-types (>= 1.16)
+    rubyzip (0.9.9)
+    simple_oauth (0.2.0)
+    twitter (4.7.0)
+      faraday (~> 0.8, < 0.10)
+      multi_json (~> 1.0)
+      simple_oauth (~> 0.2)
+    typhoeus (0.6.3)
+      ethon (~> 0.5.11)
+    unicode_utils (1.4.0)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  iron_worker_ng
+  punkt-segmenter
+  twitter
+  typhoeus