X-Git-Url: https://git.cworth.org/git?a=blobdiff_plain;f=lib%2Fsup%2Findex.rb;h=cd7c77593ea5513ca4dc300f437fa72738bbcbb4;hb=9d19facb64020cb299af59c8d8e552c90a0021d1;hp=255e2f82dad0a377476df00fac50129b2a661a16;hpb=43df7bd75d887a04b9d1433a534b7fa77188663d;p=sup

diff --git a/lib/sup/index.rb b/lib/sup/index.rb
index 255e2f8..cd7c775 100644
--- a/lib/sup/index.rb
+++ b/lib/sup/index.rb
@@ -2,36 +2,25 @@
 
 require 'thread'
 require 'fileutils'
-require_gem 'ferret', ">= 0.10.13"
+require 'ferret'
 
 module Redwood
 
-class IndexError < StandardError
-  attr_reader :source
-
-  def initialize source, s
-    super s
-    @source = source
-  end
-end
-
 class Index
   include Singleton
 
-  LOAD_THREAD_PETIT_DELAY = 0.1
-  LOAD_THREAD_GRAND_DELAY = 5
-
-  MESSAGES_AT_A_TIME = 10
-
-  attr_reader :index # debugging only
-  
+  attr_reader :index
   def initialize dir=BASE_DIR
     @dir = dir
-    @mutex = Mutex.new
-    @load_thread = nil # loads new messages
     @sources = {}
     @sources_dirty = false
 
+    wsa = Ferret::Analysis::WhiteSpaceAnalyzer.new false
+    sa = Ferret::Analysis::StandardAnalyzer.new Ferret::Analysis::FULL_ENGLISH_STOP_WORDS, true
+    @analyzer = Ferret::Analysis::PerFieldAnalyzer.new wsa
+    @analyzer[:body] = sa
+    @qparser ||= Ferret::QueryParser.new :default_field => :body, :analyzer => @analyzer
+
     self.class.i_am_the_instance self
   end
 
@@ -41,6 +30,7 @@ class Index
   end
 
   def save
+    Redwood::log "saving index and sources..."
     FileUtils.mkdir_p @dir unless File.exists? @dir
     save_sources
     save_index
@@ -50,28 +40,26 @@ class Index
     raise "duplicate source!" if @sources.include? source
     @sources_dirty = true
     source.id ||= @sources.size
-    source.id += 1 while @sources.member? source.id
+    ##TODO: why was this necessary?
+    ##source.id += 1 while @sources.member? source.id
     @sources[source.id] = source
   end
 
-  def source_for name; @sources.values.find { |s| s.is_source_for? name }; end
+  def source_for uri; @sources.values.find { |s| s.is_source_for? uri }; end
   def usual_sources; @sources.values.find_all { |s| s.usual? }; end
+  def sources; @sources.values; end
 
   def load_index dir=File.join(@dir, "ferret")
-    wsa = Ferret::Analysis::WhiteSpaceAnalyzer.new false
-    sa = Ferret::Analysis::StandardAnalyzer.new
-    analyzer = Ferret::Analysis::PerFieldAnalyzer.new wsa
-    analyzer[:body] = sa
-
     if File.exists? dir
-      Redwood::log "loading index"
-      @index = Ferret::Index::Index.new(:path => dir, :analyzer => analyzer)
+      Redwood::log "loading index..."
+      @index = Ferret::Index::Index.new(:path => dir, :analyzer => @analyzer)
+      Redwood::log "loaded index of #{@index.size} messages"
     else
-      Redwood::log "creating index"
+      Redwood::log "creating index..."
       field_infos = Ferret::Index::FieldInfos.new :store => :yes
       field_infos.add_field :message_id
       field_infos.add_field :source_id
-      field_infos.add_field :source_info, :index => :no, :term_vector => :no
+      field_infos.add_field :source_info
       field_infos.add_field :date, :index => :untokenized
       field_infos.add_field :body, :store => :no
       field_infos.add_field :label
@@ -81,26 +69,54 @@ class Index
       field_infos.add_field :refs
       field_infos.add_field :snippet, :index => :no, :term_vector => :no
       field_infos.create_index dir
-      @index = Ferret::Index::Index.new(:path => dir, :analyzer => analyzer)
+      @index = Ferret::Index::Index.new(:path => dir, :analyzer => @analyzer)
     end
   end
 
-  ## update the message by deleting and re-adding
-  def update_message m, source=nil, source_info=nil
-    docid, entry = load_entry_for_id m.id
-    if entry
-      source ||= entry[:source_id].to_i
-      source_info ||= entry[:source_info].to_i
-    end
-    raise "no entry and no source info for message #{m.id}" unless source && source_info
+  ## Syncs the message to the index: deleting if it's already there,
+  ## and adding either way. Index state will be determined by m.labels.
+  ##
+  ## docid and entry can be specified if they're already known.
+  def sync_message m, docid=nil, entry=nil
+    docid, entry = load_entry_for_id m.id unless docid && entry
+
+    raise "no source info for message #{m.id}" unless m.source && m.source_info
+    raise "trying deleting non-corresponding entry #{docid}" if docid && @index[docid][:message_id] != m.id
+
+    source_id = 
+      if m.source.is_a? Integer
+        raise "Debugging: integer source set"
+        m.source
+      else
+        m.source.id or raise "unregistered source #{m.source} (id #{m.source.id.inspect})"
+      end
+
+    to = (m.to + m.cc + m.bcc).map { |x| x.email }.join(" ")
+    d = {
+      :message_id => m.id,
+      :source_id => source_id,
+      :source_info => m.source_info,
+      :date => m.date.to_indexable_s,
+      :body => m.content,
+      :snippet => m.snippet,
+      :label => m.labels.join(" "),
+      :from => m.from ? m.from.email : "",
+      :to => (m.to + m.cc + m.bcc).map { |x| x.email }.join(" "),
+      :subject => wrap_subj(Message.normalize_subj(m.subj)),
+      :refs => (m.refs + m.replytos).uniq.join(" "),
+    }
 
-    raise "deleting non-corresponding entry #{docid}" unless @index[docid][:message_id] == m.id
-    @index.delete docid
-    add_message m
+    @index.delete docid if docid
+    @index.add_document d
+    
+    docid, entry = load_entry_for_id m.id
+    ## this hasn't been triggered in a long time. TODO: decide whether it's still a problem.
+    raise "just added message #{m.id} but couldn't find it in a search" unless docid
+    true
   end
 
   def save_index fn=File.join(@dir, "ferret")
-    # don't have to do anything apparently
+    # don't have to do anything,  apparently
   end
 
   def contains_id? id
@@ -110,11 +126,10 @@ class Index
   def size; @index.size; end
 
   ## you should probably not call this on a block that doesn't break
-  ## rather quickly because the results will probably be, as we say
-  ## in scotland, frikkin' huuuge.
+  ## rather quickly because the results can be very large.
   EACH_BY_DATE_NUM = 100
   def each_id_by_date opts={}
-    return if @index.size == 0 # otherwise ferret barfs
+    return if @index.size == 0 # otherwise ferret barfs ###TODO: remove this once my ferret patch is accepted
     query = build_query opts
     offset = 0
     while true
@@ -127,23 +142,25 @@ class Index
   end
 
   def num_results_for opts={}
-    query = build_query opts
-    x = @index.search(query).total_hits
-    Redwood::log "num_results_for: have #{x} for query #{query}"
-    x
+    return 0 if @index.size == 0 # otherwise ferret barfs ###TODO: remove this once my ferret patch is accepted
+    q = build_query opts
+    index.search(q).total_hits
   end
 
+  ## yield all messages in the thread containing 'm' by repeatedly
+  ## querying the index. yields pairs of message ids and
+  ## message-building lambdas, so that building an unwanted message
+  ## can be skipped in the block if desired.
+  ##
+  ## stops loading any thread if a message with a :killed flag is found.
   SAME_SUBJECT_DATE_LIMIT = 7
   def each_message_in_thread_for m, opts={}
+    Redwood::log "Building thread for #{m.id}: #{m.subj}"
     messages = {}
     searched = {}
     num_queries = 0
 
-    ## temporarily disabling subject searching because it's a
-    ## significant slowdown.
-    ##
-    ## TODO: make this configurable, i guess
-    if false
+    if $config[:thread_by_subject] # do subject queries
       date_min = m.date - (SAME_SUBJECT_DATE_LIMIT * 12 * 3600)
       date_max = m.date + (SAME_SUBJECT_DATE_LIMIT * 12 * 3600)
 
@@ -155,6 +172,8 @@ class Index
       q.add_query sq, :must
       q.add_query Ferret::Search::RangeQuery.new(:date, :>= => date_min.to_indexable_s, :<= => date_max.to_indexable_s), :must
 
+      q = build_query :qobj => q
+
       pending = @index.search(q).hits.map { |hit| @index[hit.doc][:message_id] }
       Redwood::log "found #{pending.size} results for subject query #{q}"
     else
@@ -169,18 +188,22 @@ class Index
       q.add_query Ferret::Search::TermQuery.new(:message_id, id), :should
       q.add_query Ferret::Search::TermQuery.new(:refs, id), :should
 
+      q = build_query :qobj => q, :load_killed => true
+
       num_queries += 1
       @index.search_each(q, :limit => :all) do |docid, score|
         break if opts[:limit] && messages.size >= opts[:limit]
+        break if @index[docid][:label].split(/\s+/).include? "killed" unless opts[:load_killed]
         mid = @index[docid][:message_id]
-        unless messages.member? mid
+        unless id == mid || messages.member?(mid)
+          Redwood::log "got #{mid} as a child of #{id}"
           messages[mid] ||= lambda { build_message docid }
           refs = @index[docid][:refs].split(" ")
           pending += refs
         end
       end
     end
-    Redwood::log "ran #{num_queries} queries to build thread of #{messages.size} messages for #{m.id}"
+    Redwood::log "ran #{num_queries} queries to build thread of #{messages.size + 1} messages for #{m.id}" if num_queries > 0
     messages.each { |mid, builder| yield mid, builder }
   end
 
@@ -190,64 +213,24 @@ class Index
     source = @sources[doc[:source_id].to_i]
     #puts "building message #{doc[:message_id]} (#{source}##{doc[:source_info]})"
     raise "invalid source #{doc[:source_id]}" unless source
-    begin
-      raise "no snippet" unless doc[:snippet]
-      Message.new source, doc[:source_info].to_i, 
-                  doc[:label].split(" ").map { |s| s.intern },
-                  doc[:snippet]
-    rescue MessageFormatError => e
-      raise IndexError.new(source, "error building message #{doc[:message_id]} at #{source}/#{doc[:source_info]}: #{e.message}")
-      nil
-    end
-  end
 
-  def start_load_thread
-    return if @load_thread
-    @load_thread = true
-    @load_thread = ::Thread.new do
-      while @load_thread
-        load_some_entries ENTRIES_AT_A_TIME, LOAD_THREAD_PETIT_DELAY, LOAD_THREAD_GRAND_DELAY
-      end
-    end
+    fake_header = {
+      "date" => Time.at(doc[:date].to_i),
+      "subject" => unwrap_subj(doc[:subject]),
+      "from" => doc[:from],
+      "to" => doc[:to],
+      "message-id" => doc[:message_id],
+      "references" => doc[:refs],
+    }
+
+    Message.new :source => source, :source_info => doc[:source_info].to_i, 
+                :labels => doc[:label].split(" ").map { |s| s.intern },
+                :snippet => doc[:snippet], :header => fake_header
   end
 
-  def end_load_thread; @load_thread = nil; end
   def fresh_thread_id; @next_thread_id += 1; end
-
   def wrap_subj subj; "__START_SUBJECT__ #{subj} __END_SUBJECT__"; end
-
-  def add_message m
-    return false if contains? m
-
-    source_id = 
-      if m.source.is_a? Integer
-        m.source
-      else
-        m.source.id or raise "unregistered source #{m.source}"
-      end
-
-    to = (m.to + m.cc + m.bcc).map { |x| x.email }.join(" ")
-    d = {
-      :message_id => m.id,
-      :source_id => source_id,
-      :source_info => m.source_info,
-      :date => m.date.to_indexable_s,
-      :body => m.content,
-      :snippet => m.snippet,
-      :label => m.labels.join(" "),
-      :from => m.from ? m.from.email : "",
-      :to => (m.to + m.cc + m.bcc).map { |x| x.email }.join(" "),
-      :subject => wrap_subj(Message.normalize_subj(m.subj)),
-      :refs => (m.refs + m.replytos).join(" "),
-    }
-
-    @index.add_document d
-    
-    ## TODO: figure out why this is sometimes triggered
-    #docid, entry = load_entry_for_id m.id
-    #raise "just added message #{m.id} but couldn't find it in a search" unless docid
-    true
-  end
+  def unwrap_subj subj; subj =~ /__START_SUBJECT__ (.*?) __END_SUBJECT__/ && $1; end
 
   def drop_entry docno; @index.delete docno; end
 
@@ -289,64 +272,46 @@ class Index
     contacts.keys.compact
   end
 
+  def load_sources fn=Redwood::SOURCE_FN
+    source_array = (Redwood::load_yaml_obj(fn) || []).map { |o| Recoverable.new o }
+    @sources = Hash[*(source_array).map { |s| [s.id, s] }.flatten]
+    @sources_dirty = false
+  end
+
 protected
 
-  ## TODO: convert this to query objects rather than strings
+  def parse_user_query_string str; @qparser.parse str; end
   def build_query opts
-    query = ""
-    query += opts[:labels].map { |t| "+label:#{t}" }.join(" ") if opts[:labels]
-    query += " +label:#{opts[:label]}" if opts[:label]
-    query += " #{opts[:content]}" if opts[:content]
+    query = Ferret::Search::BooleanQuery.new
+    query.add_query opts[:qobj], :must if opts[:qobj]
+    labels = ([opts[:label]] + (opts[:labels] || [])).compact
+    labels.each { |t| query.add_query Ferret::Search::TermQuery.new("label", t.to_s), :must }
     if opts[:participants]
-      query += "+(" + 
-        opts[:participants].map { |p| "from:#{p.email} OR to:#{p.email}" }.join(" OR ") + ")"
+      q2 = Ferret::Search::BooleanQuery.new
+      opts[:participants].each do |p|
+        q2.add_query Ferret::Search::TermQuery.new("from", p.email), :should
+        q2.add_query Ferret::Search::TermQuery.new("to", p.email), :should
+      end
+      query.add_query q2, :must
     end
         
-    query += " -label:spam" unless opts[:load_spam] || opts[:labels] == :spam || 
-      (opts[:labels] && opts[:labels].include?(:spam))
-    query += " -label:killed" unless opts[:load_killed] || opts[:labels] == :killed || 
-      (opts[:labels] && opts[:labels].include?(:killed))
+    query.add_query Ferret::Search::TermQuery.new("label", "spam"), :must_not unless opts[:load_spam] || labels.include?(:spam)
+    query.add_query Ferret::Search::TermQuery.new("label", "deleted"), :must_not unless opts[:load_deleted] || labels.include?(:deleted)
+    query.add_query Ferret::Search::TermQuery.new("label", "killed"), :must_not unless opts[:load_killed] || labels.include?(:killed)
     query
   end
 
-  def load_sources fn=Redwood::SOURCE_FN
-    @sources = Hash[*(Redwood::load_yaml_obj(fn) || []).map { |s| [s.id, s] }.flatten]
-    @sources_dirty = false
-  end
-
   def save_sources fn=Redwood::SOURCE_FN
     if @sources_dirty || @sources.any? { |id, s| s.dirty? }
-      FileUtils.mv fn, fn + ".bak", :force => true if File.exists? fn
-      Redwood::save_yaml_obj @sources.values, fn 
-    end
-    @sources_dirty = false
-  end
-
-  def load_some_entries max=ENTRIES_AT_A_TIME, delay1=nil, delay2=nil
-    num = 0
-    begin
-      @sources.each_with_index do |source, source_id|
-        next if source.done? || num >= max
-        source.each do |source_info, label|
-          begin
-            m = Message.new(source, source_info, label + [:inbox])
-            add_message m unless contains_id? m.id
-            puts m.content.inspect
-            num += 1
-          rescue MessageFormatError => e
-            $stderr.puts "ignoring erroneous message at #{source}##{source_info}: #{e.message}"
-          end
-          break if num >= max
-          sleep delay1 if delay1
-        end
-        Redwood::log "loaded #{num} entries from #{source}"
-        sleep delay2 if delay2
+      bakfn = fn + ".bak"
+      if File.exists? fn
+        File.chmod 0600, fn
+        FileUtils.mv fn, bakfn, :force => true unless File.exists?(bakfn) && File.size(bakfn) > File.size(fn)
       end
-    ensure
-      save_sources
-      save_index
+      Redwood::save_yaml_obj @sources.values, fn
+      File.chmod 0600, fn
     end
-    num
+    @sources_dirty = false
   end
 end