fixed polling (yet again)

[sup] / lib / sup / index.rb

diff --git a/lib/sup/index.rb b/lib/sup/index.rb
index faabe8dc3d98247def9639df8d12646fd433fce1..28fde2a097b4c5336c5efdbcf10ac4df9e2450bc 100644 (file)
--- a/lib/sup/index.rb
+++ b/lib/sup/index.rb
@@ -3,7 +3,6 @@
 require 'thread'
 require 'fileutils'
 require 'ferret'
-#require_gem 'ferret', ">= 0.10.13"
 
 module Redwood
 
@@ -73,24 +72,29 @@ class Index
     end
   end
 
-  ## update the message by deleting and re-adding
-  def update_message m, source=nil, source_info=nil
-    docid, entry = load_entry_for_id m.id
-    if entry
-      source ||= entry[:source_id].to_i
-      source_info ||= entry[:source_info].to_i
+  ## Update the message state on disk, by deleting and re-adding it.
+  ## The message must exist in the index. docid and entry are found
+  ## unless given.
+  ##
+  ## Overwrites the labels on disk with the new labels in 'm', so that
+  ## we can actually change message state.
+  def update_message m, docid=nil, entry=nil
+    unless docid && entry
+      docid, entry = load_entry_for_id m.id
+      raise ArgumentError, "cannot find #{m.id} in the index" unless entry
     end
 
-    ## this happens sometimes. i'm not sure why. ferret bug?
-    raise "no entry and no source info for message #{m.id}: source #{source.inspect}, info #{source_info.inspect}, entry #{entry.inspect}, query #{Ferret::Search::TermQuery.new(:message_id, m.id)}, results #{@index.search(Ferret::Search::TermQuery.new(:message_id, m.id)).inspect}" unless source && source_info
+    raise "no entry and no source info for message #{m.id}" unless m.source && m.source_info
 
     raise "deleting non-corresponding entry #{docid}" unless @index[docid][:message_id] == m.id
+
     @index.delete docid
     add_message m
+    docid, entry = load_entry_for_id m.id
   end
 
   def save_index fn=File.join(@dir, "ferret")
-    # don't have to do anything apparently
+    # don't have to do anything,  apparently
   end
 
   def contains_id? id
@@ -100,8 +104,7 @@ class Index
   def size; @index.size; end
 
   ## you should probably not call this on a block that doesn't break
-  ## rather quickly because the results will probably be, as we say
-  ## in scotland, frikkin' huuuge.
+  ## rather quickly because the results can be very large.
   EACH_BY_DATE_NUM = 100
   def each_id_by_date opts={}
     return if @index.size == 0 # otherwise ferret barfs ###TODO: remove this once my ferret patch is accepted
@@ -123,20 +126,20 @@ class Index
   end
 
   ## yield all messages in the thread containing 'm' by repeatedly
-  ## querying the index.  yields pairs of message ids and
+  ## querying the index. uields pairs of message ids and
   ## message-building lambdas, so that building an unwanted message
   ## can be skipped in the block if desired.
+  ##
+  ## stops loading any thread if a message with a :killed flag is found.
+
   SAME_SUBJECT_DATE_LIMIT = 7
   def each_message_in_thread_for m, opts={}
     messages = {}
     searched = {}
     num_queries = 0
 
-    ## temporarily disabling subject searching because it's a
-    ## significant slowdown.
-    ##
-    ## TODO: make this configurable, i guess
-    if true
+    ## todo: make subject querying configurable
+    if true # do subject queries
       date_min = m.date - (SAME_SUBJECT_DATE_LIMIT * 12 * 3600)
       date_max = m.date + (SAME_SUBJECT_DATE_LIMIT * 12 * 3600)
 
@@ -146,9 +149,10 @@ class Index
         sq.add_term t
       end
       q.add_query sq, :must
-      q.add_query Ferret::Search::TermQuery.new(:label, "spam"), :must_not
       q.add_query Ferret::Search::RangeQuery.new(:date, :>= => date_min.to_indexable_s, :<= => date_max.to_indexable_s), :must
 
+      q = build_query :qobj => q
+
       pending = @index.search(q).hits.map { |hit| @index[hit.doc][:message_id] }
       Redwood::log "found #{pending.size} results for subject query #{q}"
     else
@@ -163,9 +167,12 @@ class Index
       q.add_query Ferret::Search::TermQuery.new(:message_id, id), :should
       q.add_query Ferret::Search::TermQuery.new(:refs, id), :should
 
+      q = build_query :qobj => q, :load_killed => true
+
       num_queries += 1
       @index.search_each(q, :limit => :all) do |docid, score|
         break if opts[:limit] && messages.size >= opts[:limit]
+        break if @index[docid][:label].split(/\s+/).include? "killed" unless opts[:load_killed]
         mid = @index[docid][:message_id]
         unless messages.member? mid
           messages[mid] ||= lambda { build_message docid }
@@ -203,8 +210,10 @@ class Index
   def wrap_subj subj; "__START_SUBJECT__ #{subj} __END_SUBJECT__"; end
   def unwrap_subj subj; subj =~ /__START_SUBJECT__ (.*?) __END_SUBJECT__/ && $1; end
 
+  ## Adds a message to the index. The message cannot already exist in
+  ## the index.
   def add_message m
-    return false if contains? m
+    raise ArgumentError, "index already contains #{m.id}" if contains? m
 
     source_id = 
       if m.source.is_a? Integer
@@ -230,8 +239,8 @@ class Index
 
     @index.add_document d
     
-    ## TODO: figure out why this is sometimes triggered
     docid, entry = load_entry_for_id m.id
+    ## this hasn't been triggered in a long time. TODO: decide whether it's still a problem.
     raise "just added message #{m.id} but couldn't find it in a search" unless docid
     true
   end
@@ -294,6 +303,7 @@ protected
     end
         
     query.add_query Ferret::Search::TermQuery.new("label", "spam"), :must_not unless opts[:load_spam] || labels.include?(:spam)
+    query.add_query Ferret::Search::TermQuery.new("label", "deleted"), :must_not unless opts[:load_deleted] || labels.include?(:deleted)
     query.add_query Ferret::Search::TermQuery.new("label", "killed"), :must_not unless opts[:load_killed] || labels.include?(:killed)
     query
   end