mbox::loader#total went missing for some reason. replaced.

re-fixed tokenization in index.rb to do downcasing. i think that's probably
the right idea for email search.

git-svn-id: svn://rubyforge.org/var/svn/sup/trunk@105 5c8cc53c-5e98-4d25-b20a-d8db53a31250

diff --git a/lib/sup/index.rb b/lib/sup/index.rb
index b2b9441094b99f776783892b95c7edebb86cd64b..3151f3b43719f4b60750c6b264c26074b81fe2d6 100644 (file)
--- a/lib/sup/index.rb
+++ b/lib/sup/index.rb
@@ -2,8 +2,8 @@
 
 require 'thread'
 require 'fileutils'
-#require 'ferret'
-require_gem 'ferret', ">= 0.10.13"
+require 'ferret'
+#require_gem 'ferret', ">= 0.10.13"
 
 module Redwood
 
@@ -26,7 +26,7 @@ class Index
     @sources = {}
     @sources_dirty = false
 
-    wsa = Ferret::Analysis::WhiteSpaceAnalyzer.new true
+    wsa = Ferret::Analysis::WhiteSpaceAnalyzer.new false
     sa = Ferret::Analysis::StandardAnalyzer.new Ferret::Analysis::FULL_ENGLISH_STOP_WORDS, true
     @analyzer = Ferret::Analysis::PerFieldAnalyzer.new wsa
     @analyzer[:body] = sa
@@ -90,7 +90,7 @@ class Index
     end
 
     ## this happens sometimes. i'm not sure why. ferret bug?
-    raise "no entry and no source info for message #{m.id}" unless source && source_info
+    raise "no entry and no source info for message #{m.id}: source #{source.inspect}, info #{source_info.inspect}, entry #{entry.inspect}, query #{Ferret::Search::TermQuery.new(:message_id, m.id)}, results #{@index.search(Ferret::Search::TermQuery.new(:message_id, m.id)).inspect}" unless source && source_info
 
     raise "deleting non-corresponding entry #{docid}" unless @index[docid][:message_id] == m.id
     @index.delete docid

diff --git a/lib/sup/mbox/loader.rb b/lib/sup/mbox/loader.rb
index ec7de85258618745fe54e622a6a592bb330e9aa8..a459539296532ada7a81d36a5acd6deea36a61a7 100644 (file)
--- a/lib/sup/mbox/loader.rb
+++ b/lib/sup/mbox/loader.rb
@@ -29,6 +29,7 @@ class Loader < Source
 
   def start_offset; 0; end
   def end_offset; File.size @f; end
+  def total; end_offset; end
 
   def load_header offset
     header = nil