X-Git-Url: https://git.cworth.org/git?a=blobdiff_plain;f=lib%2Fsup%2Findex.rb;h=28fde2a097b4c5336c5efdbcf10ac4df9e2450bc;hb=07b1fdc67e634001857aa5a252f5a3a31124bb3b;hp=3de011e9e4be2fcfcc6825b625351dcb9bdcab67;hpb=35e74b8516a33594e59f973d0e0858670010aa97;p=sup

diff --git a/lib/sup/index.rb b/lib/sup/index.rb
index 3de011e..28fde2a 100644
--- a/lib/sup/index.rb
+++ b/lib/sup/index.rb
@@ -3,24 +3,13 @@
 require 'thread'
 require 'fileutils'
 require 'ferret'
-#require_gem 'ferret', ">= 0.10.13"
 
 module Redwood
 
-class IndexError < StandardError
-  attr_reader :source
-
-  def initialize source, s
-    super s
-    @source = source
-  end
-end
-
 class Index
   include Singleton
 
-  attr_reader :index # debugging only
-  
+  attr_reader :index
   def initialize dir=BASE_DIR
     @dir = dir
     @sources = {}
@@ -55,8 +44,9 @@ class Index
     @sources[source.id] = source
   end
 
-  def source_for name; @sources.values.find { |s| s.is_source_for? name }; end
+  def source_for uri; @sources.values.find { |s| s.is_source_for? uri }; end
   def usual_sources; @sources.values.find_all { |s| s.usual? }; end
+  def sources; @sources.values; end
 
   def load_index dir=File.join(@dir, "ferret")
     if File.exists? dir
@@ -82,24 +72,29 @@ class Index
     end
   end
 
-  ## update the message by deleting and re-adding
-  def update_message m, source=nil, source_info=nil
-    docid, entry = load_entry_for_id m.id
-    if entry
-      source ||= entry[:source_id].to_i
-      source_info ||= entry[:source_info].to_i
+  ## Update the message state on disk, by deleting and re-adding it.
+  ## The message must exist in the index. docid and entry are found
+  ## unless given.
+  ##
+  ## Overwrites the labels on disk with the new labels in 'm', so that
+  ## we can actually change message state.
+  def update_message m, docid=nil, entry=nil
+    unless docid && entry
+      docid, entry = load_entry_for_id m.id
+      raise ArgumentError, "cannot find #{m.id} in the index" unless entry
     end
 
-    ## this happens sometimes. i'm not sure why. ferret bug?
-    raise "no entry and no source info for message #{m.id}: source #{source.inspect}, info #{source_info.inspect}, entry #{entry.inspect}, query #{Ferret::Search::TermQuery.new(:message_id, m.id)}, results #{@index.search(Ferret::Search::TermQuery.new(:message_id, m.id)).inspect}" unless source && source_info
+    raise "no entry and no source info for message #{m.id}" unless m.source && m.source_info
 
     raise "deleting non-corresponding entry #{docid}" unless @index[docid][:message_id] == m.id
+
     @index.delete docid
     add_message m
+    docid, entry = load_entry_for_id m.id
   end
 
   def save_index fn=File.join(@dir, "ferret")
-    # don't have to do anything apparently
+    # don't have to do anything,  apparently
   end
 
   def contains_id? id
@@ -109,8 +104,7 @@ class Index
   def size; @index.size; end
 
   ## you should probably not call this on a block that doesn't break
-  ## rather quickly because the results will probably be, as we say
-  ## in scotland, frikkin' huuuge.
+  ## rather quickly because the results can be very large.
   EACH_BY_DATE_NUM = 100
   def each_id_by_date opts={}
     return if @index.size == 0 # otherwise ferret barfs ###TODO: remove this once my ferret patch is accepted
@@ -132,20 +126,20 @@ class Index
   end
 
   ## yield all messages in the thread containing 'm' by repeatedly
-  ## querying the index.  yields pairs of message ids and
+  ## querying the index. uields pairs of message ids and
   ## message-building lambdas, so that building an unwanted message
   ## can be skipped in the block if desired.
+  ##
+  ## stops loading any thread if a message with a :killed flag is found.
+
   SAME_SUBJECT_DATE_LIMIT = 7
   def each_message_in_thread_for m, opts={}
     messages = {}
     searched = {}
     num_queries = 0
 
-    ## temporarily disabling subject searching because it's a
-    ## significant slowdown.
-    ##
-    ## TODO: make this configurable, i guess
-    if false
+    ## todo: make subject querying configurable
+    if true # do subject queries
       date_min = m.date - (SAME_SUBJECT_DATE_LIMIT * 12 * 3600)
       date_max = m.date + (SAME_SUBJECT_DATE_LIMIT * 12 * 3600)
 
@@ -157,6 +151,8 @@ class Index
       q.add_query sq, :must
       q.add_query Ferret::Search::RangeQuery.new(:date, :>= => date_min.to_indexable_s, :<= => date_max.to_indexable_s), :must
 
+      q = build_query :qobj => q
+
       pending = @index.search(q).hits.map { |hit| @index[hit.doc][:message_id] }
       Redwood::log "found #{pending.size} results for subject query #{q}"
     else
@@ -171,9 +167,12 @@ class Index
       q.add_query Ferret::Search::TermQuery.new(:message_id, id), :should
       q.add_query Ferret::Search::TermQuery.new(:refs, id), :should
 
+      q = build_query :qobj => q, :load_killed => true
+
       num_queries += 1
       @index.search_each(q, :limit => :all) do |docid, score|
         break if opts[:limit] && messages.size >= opts[:limit]
+        break if @index[docid][:label].split(/\s+/).include? "killed" unless opts[:load_killed]
         mid = @index[docid][:message_id]
         unless messages.member? mid
           messages[mid] ||= lambda { build_message docid }
@@ -182,7 +181,7 @@ class Index
         end
       end
     end
-    Redwood::log "ran #{num_queries} queries to build thread of #{messages.size} messages for #{m.id}"
+    Redwood::log "ran #{num_queries} queries to build thread of #{messages.size} messages for #{m.id}" if num_queries > 0
     messages.each { |mid, builder| yield mid, builder }
   end
 
@@ -211,14 +210,16 @@ class Index
   def wrap_subj subj; "__START_SUBJECT__ #{subj} __END_SUBJECT__"; end
   def unwrap_subj subj; subj =~ /__START_SUBJECT__ (.*?) __END_SUBJECT__/ && $1; end
 
+  ## Adds a message to the index. The message cannot already exist in
+  ## the index.
   def add_message m
-    return false if contains? m
+    raise ArgumentError, "index already contains #{m.id}" if contains? m
 
     source_id = 
       if m.source.is_a? Integer
         m.source
       else
-        m.source.id or raise "unregistered source #{m.source}"
+        m.source.id or raise "unregistered source #{m.source} (id #{m.source.id.inspect})"
       end
 
     to = (m.to + m.cc + m.bcc).map { |x| x.email }.join(" ")
@@ -238,9 +239,9 @@ class Index
 
     @index.add_document d
     
-    ## TODO: figure out why this is sometimes triggered
-    #docid, entry = load_entry_for_id m.id
-    #raise "just added message #{m.id} but couldn't find it in a search" unless docid
+    docid, entry = load_entry_for_id m.id
+    ## this hasn't been triggered in a long time. TODO: decide whether it's still a problem.
+    raise "just added message #{m.id} but couldn't find it in a search" unless docid
     true
   end
 
@@ -288,7 +289,6 @@ protected
 
   def parse_user_query_string str; @qparser.parse str; end
   def build_query opts
-
     query = Ferret::Search::BooleanQuery.new
     query.add_query opts[:qobj], :must if opts[:qobj]
     labels = ([opts[:label]] + (opts[:labels] || [])).compact
@@ -303,6 +303,7 @@ protected
     end
         
     query.add_query Ferret::Search::TermQuery.new("label", "spam"), :must_not unless opts[:load_spam] || labels.include?(:spam)
+    query.add_query Ferret::Search::TermQuery.new("label", "deleted"), :must_not unless opts[:load_deleted] || labels.include?(:deleted)
     query.add_query Ferret::Search::TermQuery.new("label", "killed"), :must_not unless opts[:load_killed] || labels.include?(:killed)
     query
   end