Merge branch 'various-api-refactors' into next

[sup] / lib / sup / xapian_index.rb

diff --git a/lib/sup/xapian_index.rb b/lib/sup/xapian_index.rb
index 7bbc41c3e48497ad96d8e3cd3bfcdbe97644f8ec..dbf66431bc263ada5420b9a981f766c9ca5675d4 100644 (file)
--- a/lib/sup/xapian_index.rb
+++ b/lib/sup/xapian_index.rb
@@ -14,20 +14,22 @@ class XapianIndex < BaseIndex
   ## so we must ensure they're reasonably valid. this typically only affect
   ## spam.
   MIN_DATE = Time.at 0
-  MAX_DATE = Time.at(2**31)
+  MAX_DATE = Time.at(2**31-1)
 
   def initialize dir=BASE_DIR
     super
 
     @index_mutex = Monitor.new
+  end
 
-    @entries = MarshalledGDBM.new File.join(dir, "entries.db")
-    @docids = MarshalledGDBM.new File.join(dir, "docids.db")
-    @thread_members = MarshalledGDBM.new File.join(dir, "thread_members.db")
-    @thread_ids = MarshalledGDBM.new File.join(dir, "thread_ids.db")
-    @assigned_docids = GDBM.new File.join(dir, "assigned_docids.db")
+  def load_index
+    @entries = MarshalledGDBM.new File.join(@dir, "entries.db")
+    @docids = MarshalledGDBM.new File.join(@dir, "docids.db")
+    @thread_members = MarshalledGDBM.new File.join(@dir, "thread_members.db")
+    @thread_ids = MarshalledGDBM.new File.join(@dir, "thread_ids.db")
+    @assigned_docids = GDBM.new File.join(@dir, "assigned_docids.db")
 
-    @xapian = Xapian::WritableDatabase.new(File.join(dir, "xapian"), Xapian::DB_CREATE_OR_OPEN)
+    @xapian = Xapian::WritableDatabase.new(File.join(@dir, "xapian"), Xapian::DB_CREATE_OR_OPEN)
     @term_generator = Xapian::TermGenerator.new()
     @term_generator.stemmer = Xapian::Stem.new(STEM_LANGUAGE)
     @enquire = Xapian::Enquire.new @xapian
@@ -35,9 +37,6 @@ class XapianIndex < BaseIndex
     @enquire.docid_order = Xapian::Enquire::ASCENDING
   end
 
-  def load_index
-  end
-
   def save_index
   end
 
@@ -74,9 +73,9 @@ class XapianIndex < BaseIndex
       'date' => Time.at(entry[:date]),
       'subject' => entry[:subject],
       'from' => mk_addrs[[entry[:from]]],
-      'to' => mk_addrs[[entry[:to]]],
-      'cc' => mk_addrs[[entry[:cc]]],
-      'bcc' => mk_addrs[[entry[:bcc]]],
+      'to' => mk_addrs[entry[:to]],
+      'cc' => mk_addrs[entry[:cc]],
+      'bcc' => mk_addrs[entry[:bcc]],
       'reply-tos' => mk_refs[entry[:replytos]],
       'references' => mk_refs[entry[:refs]],
      }
@@ -88,6 +87,10 @@ class XapianIndex < BaseIndex
       m
   end
 
+  def add_message m; sync_message m end
+  def update_message m; sync_message m end
+  def update_message_state m; sync_message m end
+
   def sync_message m, opts={}
     entry = synchronize { @entries[m.id] }
     snippet = m.snippet
@@ -101,7 +104,7 @@ class XapianIndex < BaseIndex
       :source_info => m.source_info,
       :date => (entry[:date] || m.date),
       :snippet => snippet,
-      :labels => labels.uniq,
+      :labels => labels,
       :from => (entry[:from] || [m.from.email, m.from.name]),
       :to => (entry[:to] || m.to.map { |p| [p.email, p.name] }),
       :cc => (entry[:cc] || m.cc.map { |p| [p.email, p.name] }),
@@ -111,7 +114,7 @@ class XapianIndex < BaseIndex
       :replytos => (entry[:replytos] || m.replytos),
     }
 
-    m.labels.each { |l| LabelManager << l }
+    labels.each { |l| LabelManager << l }
 
     synchronize do
       index_message m, opts
@@ -120,6 +123,7 @@ class XapianIndex < BaseIndex
     end
     true
   end
+  private :sync_message
 
   def num_results_for query={}
     xapian_query = build_xapian_query query
@@ -217,10 +221,10 @@ class XapianIndex < BaseIndex
       field, name = $1, ($3 || $4)
       case field
       when "filename"
-        Redwood::log "filename - translated #{field}:#{name} to attachment:\"#{name.downcase}\""
+        debug "filename: translated #{field}:#{name} to attachment:\"#{name.downcase}\""
         "attachment:\"#{name.downcase}\""
       when "filetype"
-        Redwood::log "filetype - translated #{field}:#{name} to attachment_extension:#{name.downcase}"
+        debug "filetype: translated #{field}:#{name} to attachment_extension:#{name.downcase}"
         "attachment_extension:#{name.downcase}"
       end
     end
@@ -234,13 +238,13 @@ class XapianIndex < BaseIndex
         if realdate
           case field
           when "after"
-            Redwood::log "chronic: translated #{field}:#{datestr} to #{realdate.end}"
+            debug "chronic: translated #{field}:#{datestr} to #{realdate.end}"
             "date:#{realdate.end.to_i}..#{lastdate}"
           when "before"
-            Redwood::log "chronic: translated #{field}:#{datestr} to #{realdate.begin}"
+            debug "chronic: translated #{field}:#{datestr} to #{realdate.begin}"
             "date:#{firstdate}..#{realdate.end.to_i}"
           else
-            Redwood::log "chronic: translated #{field}:#{datestr} to #{realdate}"
+            debug "chronic: translated #{field}:#{datestr} to #{realdate}"
             "date:#{realdate.begin.to_i}..#{realdate.end.to_i}"
           end
         else
@@ -304,6 +308,8 @@ class XapianIndex < BaseIndex
 
   DATE_VALUENO = 0
 
+  MAX_TERM_LENGTH = 245
+
   # Xapian can very efficiently sort in ascending docid order. Sup always wants
   # to sort by descending date, so this method maps between them. In order to
   # handle multiple messages per second, we use a logistic curve centered
@@ -407,10 +413,10 @@ class XapianIndex < BaseIndex
     m.attachments.each { |a| text << [a, PREFIX['attachment']] }
 
     truncated_date = if m.date < MIN_DATE
-      Redwood::log "warning: adjusting too-low date #{m.date} for indexing"
+      debug "warning: adjusting too-low date #{m.date} for indexing"
       MIN_DATE
     elsif m.date > MAX_DATE
-      Redwood::log "warning: adjusting too-high date #{m.date} for indexing"
+      debug "warning: adjusting too-high date #{m.date} for indexing"
       MAX_DATE
     else
       m.date
@@ -428,7 +434,7 @@ class XapianIndex < BaseIndex
 
     @term_generator.document = doc
     text.each { |text,prefix| @term_generator.index_text text, 1, prefix }
-    terms.each { |term| doc.add_term term }
+    terms.each { |term| doc.add_term term if term.length <= MAX_TERM_LENGTH }
     doc.add_value DATE_VALUENO, date_value
     doc.data = m.id