fixed polling (yet again)

[sup] / lib / sup / index.rb

diff --git a/lib/sup/index.rb b/lib/sup/index.rb
index cb56efe077ac6d5dd86e619a7c072ea197d98e6f..28fde2a097b4c5336c5efdbcf10ac4df9e2450bc 100644 (file)
--- a/lib/sup/index.rb
+++ b/lib/sup/index.rb
@@ -75,6 +75,9 @@ class Index
   ## Update the message state on disk, by deleting and re-adding it.
   ## The message must exist in the index. docid and entry are found
   ## unless given.
+  ##
+  ## Overwrites the labels on disk with the new labels in 'm', so that
+  ## we can actually change message state.
   def update_message m, docid=nil, entry=nil
     unless docid && entry
       docid, entry = load_entry_for_id m.id
@@ -87,48 +90,7 @@ class Index
 
     @index.delete docid
     add_message m
-  end
-
-  ## for each new message form the source, yields a bunch of stuff,
-  ## gets the message back from the block, and adds it or updates it.
-  def add_new_messages_from source
-    found = {}
-    return if source.done? || source.broken?
-
-    source.each do |offset, labels|
-      if source.broken?
-        Redwood::log "error loading messages from #{source}: #{source.broken_msg}"
-        return
-      end
-      
-      labels.each { |l| LabelManager << l }
-
-      begin
-        m = Message.new :source => source, :source_info => offset, :labels => labels
-        if found[m.id]
-          puts "skipping duplicate message #{m.id}"
-          next
-        else
-          found[m.id] = true
-        end
-
-        if m.source_marked_read?
-          m.remove_label :unread
-          labels.delete :unread
-        end
-
-        docid, entry = load_entry_for_id m.id
-        m = yield m, offset, labels, entry
-        next unless m
-        if entry
-          update_message m, docid, entry
-        else
-          add_message m
-        end
-      rescue MessageFormatError, SourceError => e
-        Redwood::log "ignoring erroneous message at #{source}##{offset}: #{e.message}"
-      end
-    end
+    docid, entry = load_entry_for_id m.id
   end
 
   def save_index fn=File.join(@dir, "ferret")
@@ -142,8 +104,7 @@ class Index
   def size; @index.size; end
 
   ## you should probably not call this on a block that doesn't break
-  ## rather quickly because the results will probably be, as we say
-  ## in scotland, frikkin' huuuge.
+  ## rather quickly because the results can be very large.
   EACH_BY_DATE_NUM = 100
   def each_id_by_date opts={}
     return if @index.size == 0 # otherwise ferret barfs ###TODO: remove this once my ferret patch is accepted
@@ -165,20 +126,20 @@ class Index
   end
 
   ## yield all messages in the thread containing 'm' by repeatedly
-  ## querying the index.  yields pairs of message ids and
+  ## querying the index. uields pairs of message ids and
   ## message-building lambdas, so that building an unwanted message
   ## can be skipped in the block if desired.
+  ##
+  ## stops loading any thread if a message with a :killed flag is found.
+
   SAME_SUBJECT_DATE_LIMIT = 7
   def each_message_in_thread_for m, opts={}
     messages = {}
     searched = {}
     num_queries = 0
 
-    ## temporarily disabling subject searching because it's a
-    ## significant slowdown.
-    ##
-    ## TODO: make this configurable, i guess
-    if true
+    ## todo: make subject querying configurable
+    if true # do subject queries
       date_min = m.date - (SAME_SUBJECT_DATE_LIMIT * 12 * 3600)
       date_max = m.date + (SAME_SUBJECT_DATE_LIMIT * 12 * 3600)
 
@@ -188,9 +149,10 @@ class Index
         sq.add_term t
       end
       q.add_query sq, :must
-      q.add_query Ferret::Search::TermQuery.new(:label, "spam"), :must_not
       q.add_query Ferret::Search::RangeQuery.new(:date, :>= => date_min.to_indexable_s, :<= => date_max.to_indexable_s), :must
 
+      q = build_query :qobj => q
+
       pending = @index.search(q).hits.map { |hit| @index[hit.doc][:message_id] }
       Redwood::log "found #{pending.size} results for subject query #{q}"
     else
@@ -205,9 +167,12 @@ class Index
       q.add_query Ferret::Search::TermQuery.new(:message_id, id), :should
       q.add_query Ferret::Search::TermQuery.new(:refs, id), :should
 
+      q = build_query :qobj => q, :load_killed => true
+
       num_queries += 1
       @index.search_each(q, :limit => :all) do |docid, score|
         break if opts[:limit] && messages.size >= opts[:limit]
+        break if @index[docid][:label].split(/\s+/).include? "killed" unless opts[:load_killed]
         mid = @index[docid][:message_id]
         unless messages.member? mid
           messages[mid] ||= lambda { build_message docid }