X-Git-Url: https://git.cworth.org/git?a=blobdiff_plain;f=lib%2Fsup%2Findex.rb;h=19d4ed6a515d3f57e8cf0ce4548243676d2924cf;hb=210b30dd62b033262ac01888947619e73e578d54;hp=2dc2101ed64313361d16cc392d85176967506f7b;hpb=612ab3c4819ad549bbd738770763186cdfb62caf;p=sup diff --git a/lib/sup/index.rb b/lib/sup/index.rb index 2dc2101..19d4ed6 100644 --- a/lib/sup/index.rb +++ b/lib/sup/index.rb @@ -2,30 +2,25 @@ require 'thread' require 'fileutils' -require_gem 'ferret', ">= 0.10.13" +require 'ferret' module Redwood -class IndexError < StandardError - attr_reader :source - - def initialize source, s - super s - @source = source - end -end - class Index include Singleton - attr_reader :index # debugging only - + attr_reader :index def initialize dir=BASE_DIR @dir = dir - @mutex = Mutex.new @sources = {} @sources_dirty = false + wsa = Ferret::Analysis::WhiteSpaceAnalyzer.new false + sa = Ferret::Analysis::StandardAnalyzer.new Ferret::Analysis::FULL_ENGLISH_STOP_WORDS, true + @analyzer = Ferret::Analysis::PerFieldAnalyzer.new wsa + @analyzer[:body] = sa + @qparser ||= Ferret::QueryParser.new :default_field => :body, :analyzer => @analyzer + self.class.i_am_the_instance self end @@ -35,6 +30,7 @@ class Index end def save + Redwood::log "saving index and sources..." FileUtils.mkdir_p @dir unless File.exists? @dir save_sources save_index @@ -49,20 +45,17 @@ class Index @sources[source.id] = source end - def source_for name; @sources.values.find { |s| s.is_source_for? name }; end + def source_for uri; @sources.values.find { |s| s.is_source_for? uri }; end def usual_sources; @sources.values.find_all { |s| s.usual? }; end + def sources; @sources.values; end def load_index dir=File.join(@dir, "ferret") - wsa = Ferret::Analysis::WhiteSpaceAnalyzer.new false - sa = Ferret::Analysis::StandardAnalyzer.new - analyzer = Ferret::Analysis::PerFieldAnalyzer.new wsa - analyzer[:body] = sa - if File.exists? dir - Redwood::log "loading index" - @index = Ferret::Index::Index.new(:path => dir, :analyzer => analyzer) + Redwood::log "loading index..." + @index = Ferret::Index::Index.new(:path => dir, :analyzer => @analyzer) + Redwood::log "loaded index of #{@index.size} messages" else - Redwood::log "creating index" + Redwood::log "creating index..." field_infos = Ferret::Index::FieldInfos.new :store => :yes field_infos.add_field :message_id field_infos.add_field :source_id @@ -76,26 +69,54 @@ class Index field_infos.add_field :refs field_infos.add_field :snippet, :index => :no, :term_vector => :no field_infos.create_index dir - @index = Ferret::Index::Index.new(:path => dir, :analyzer => analyzer) + @index = Ferret::Index::Index.new(:path => dir, :analyzer => @analyzer) end end - ## update the message by deleting and re-adding - def update_message m, source=nil, source_info=nil - docid, entry = load_entry_for_id m.id - if entry - source ||= entry[:source_id].to_i - source_info ||= entry[:source_info].to_i - end - raise "no entry and no source info for message #{m.id}" unless source && source_info + ## Syncs the message to the index: deleting if it's already there, + ## and adding either way. Index state will be determined by m.labels. + ## + ## docid and entry can be specified if they're already known. + def sync_message m, docid=nil, entry=nil + docid, entry = load_entry_for_id m.id unless docid && entry + + raise "no source info for message #{m.id}" unless m.source && m.source_info + raise "trying deleting non-corresponding entry #{docid}" if docid && @index[docid][:message_id] != m.id - raise "deleting non-corresponding entry #{docid}" unless @index[docid][:message_id] == m.id - @index.delete docid - add_message m + source_id = + if m.source.is_a? Integer + raise "Debugging: integer source set" + m.source + else + m.source.id or raise "unregistered source #{m.source} (id #{m.source.id.inspect})" + end + + to = (m.to + m.cc + m.bcc).map { |x| x.email }.join(" ") + d = { + :message_id => m.id, + :source_id => source_id, + :source_info => m.source_info, + :date => m.date.to_indexable_s, + :body => m.content, + :snippet => m.snippet, + :label => m.labels.join(" "), + :from => m.from ? m.from.email : "", + :to => (m.to + m.cc + m.bcc).map { |x| x.email }.join(" "), + :subject => wrap_subj(Message.normalize_subj(m.subj)), + :refs => (m.refs + m.replytos).uniq.join(" "), + } + + @index.delete docid if docid + @index.add_document d + + docid, entry = load_entry_for_id m.id + ## this hasn't been triggered in a long time. TODO: decide whether it's still a problem. + raise "just added message #{m.id} but couldn't find it in a search" unless docid + true end def save_index fn=File.join(@dir, "ferret") - # don't have to do anything apparently + # don't have to do anything, apparently end def contains_id? id @@ -105,11 +126,10 @@ class Index def size; @index.size; end ## you should probably not call this on a block that doesn't break - ## rather quickly because the results will probably be, as we say - ## in scotland, frikkin' huuuge. + ## rather quickly because the results can be very large. EACH_BY_DATE_NUM = 100 def each_id_by_date opts={} - return if @index.size == 0 # otherwise ferret barfs + return if @index.size == 0 # otherwise ferret barfs ###TODO: remove this once my ferret patch is accepted query = build_query opts offset = 0 while true @@ -122,23 +142,26 @@ class Index end def num_results_for opts={} - query = build_query opts - x = @index.search(query).total_hits - Redwood::log "num_results_for: have #{x} for query #{query}" - x + return 0 if @index.size == 0 # otherwise ferret barfs ###TODO: remove this once my ferret patch is accepted + q = build_query opts + index.search(q).total_hits end + ## yield all messages in the thread containing 'm' by repeatedly + ## querying the index. uields pairs of message ids and + ## message-building lambdas, so that building an unwanted message + ## can be skipped in the block if desired. + ## + ## stops loading any thread if a message with a :killed flag is found. + SAME_SUBJECT_DATE_LIMIT = 7 def each_message_in_thread_for m, opts={} messages = {} searched = {} num_queries = 0 - ## temporarily disabling subject searching because it's a - ## significant slowdown. - ## - ## TODO: make this configurable, i guess - if false + ## todo: make subject querying configurable + if true # do subject queries date_min = m.date - (SAME_SUBJECT_DATE_LIMIT * 12 * 3600) date_max = m.date + (SAME_SUBJECT_DATE_LIMIT * 12 * 3600) @@ -150,6 +173,8 @@ class Index q.add_query sq, :must q.add_query Ferret::Search::RangeQuery.new(:date, :>= => date_min.to_indexable_s, :<= => date_max.to_indexable_s), :must + q = build_query :qobj => q + pending = @index.search(q).hits.map { |hit| @index[hit.doc][:message_id] } Redwood::log "found #{pending.size} results for subject query #{q}" else @@ -164,9 +189,12 @@ class Index q.add_query Ferret::Search::TermQuery.new(:message_id, id), :should q.add_query Ferret::Search::TermQuery.new(:refs, id), :should + q = build_query :qobj => q, :load_killed => true + num_queries += 1 @index.search_each(q, :limit => :all) do |docid, score| break if opts[:limit] && messages.size >= opts[:limit] + break if @index[docid][:label].split(/\s+/).include? "killed" unless opts[:load_killed] mid = @index[docid][:message_id] unless messages.member? mid messages[mid] ||= lambda { build_message docid } @@ -175,7 +203,7 @@ class Index end end end - Redwood::log "ran #{num_queries} queries to build thread of #{messages.size} messages for #{m.id}" + Redwood::log "ran #{num_queries} queries to build thread of #{messages.size} messages for #{m.id}" if num_queries > 0 messages.each { |mid, builder| yield mid, builder } end @@ -186,83 +214,24 @@ class Index #puts "building message #{doc[:message_id]} (#{source}##{doc[:source_info]})" raise "invalid source #{doc[:source_id]}" unless source - m = - if source.broken? - nil - else - begin - Message.new :source => source, :source_info => doc[:source_info].to_i, - :labels => doc[:label].split(" ").map { |s| s.intern }, - :snippet => doc[:snippet] - rescue MessageFormatError => e - raise IndexError.new(source, "error building message #{doc[:message_id]} at #{source}/#{doc[:source_info]}: #{e.message}") - rescue SourceError => e - nil - end - end + fake_header = { + "date" => Time.at(doc[:date].to_i), + "subject" => unwrap_subj(doc[:subject]), + "from" => doc[:from], + "to" => doc[:to], + "message-id" => doc[:message_id], + "references" => doc[:refs], + } - unless m - fake_header = { - "date" => Time.at(doc[:date].to_i), - "subject" => unwrap_subj(doc[:subject]), - "from" => doc[:from], - "to" => doc[:to], - "message-id" => doc[:message_id], - "references" => doc[:refs], - } - - m = Message.new :labels => doc[:label].split(" ").map { |s| s.intern }, - :snippet => doc[:snippet], :header => fake_header, - :body => < source, :source_info => doc[:source_info].to_i, + :labels => doc[:label].split(" ").map { |s| s.intern }, + :snippet => doc[:snippet], :header => fake_header end def fresh_thread_id; @next_thread_id += 1; end def wrap_subj subj; "__START_SUBJECT__ #{subj} __END_SUBJECT__"; end def unwrap_subj subj; subj =~ /__START_SUBJECT__ (.*?) __END_SUBJECT__/ && $1; end - def add_message m - return false if contains? m - - source_id = - if m.source.is_a? Integer - m.source - else - m.source.id or raise "unregistered source #{m.source}" - end - - to = (m.to + m.cc + m.bcc).map { |x| x.email }.join(" ") - d = { - :message_id => m.id, - :source_id => source_id, - :source_info => m.source_info, - :date => m.date.to_indexable_s, - :body => m.content, - :snippet => m.snippet, - :label => m.labels.join(" "), - :from => m.from ? m.from.email : "", - :to => (m.to + m.cc + m.bcc).map { |x| x.email }.join(" "), - :subject => wrap_subj(Message.normalize_subj(m.subj)), - :refs => (m.refs + m.replytos).uniq.join(" "), - } - - @index.add_document d - - ## TODO: figure out why this is sometimes triggered - #docid, entry = load_entry_for_id m.id - #raise "just added message #{m.id} but couldn't find it in a search" unless docid - true - end - def drop_entry docno; @index.delete docno; end def load_entry_for_id mid @@ -303,31 +272,35 @@ EOS contacts.keys.compact end + def load_sources fn=Redwood::SOURCE_FN + source_array = (Redwood::load_yaml_obj(fn) || []).map { |o| Recoverable.new o } + @sources = Hash[*(source_array).map { |s| [s.id, s] }.flatten] + @sources_dirty = false + end + protected - ## TODO: convert this to query objects rather than strings + def parse_user_query_string str; @qparser.parse str; end def build_query opts - query = "" - query += opts[:labels].map { |t| "+label:#{t}" }.join(" ") if opts[:labels] - query += " +label:#{opts[:label]}" if opts[:label] - query += " #{opts[:content]}" if opts[:content] + query = Ferret::Search::BooleanQuery.new + query.add_query opts[:qobj], :must if opts[:qobj] + labels = ([opts[:label]] + (opts[:labels] || [])).compact + labels.each { |t| query.add_query Ferret::Search::TermQuery.new("label", t.to_s), :must } if opts[:participants] - query += "+(" + - opts[:participants].map { |p| "from:#{p.email} OR to:#{p.email}" }.join(" OR ") + ")" + q2 = Ferret::Search::BooleanQuery.new + opts[:participants].each do |p| + q2.add_query Ferret::Search::TermQuery.new("from", p.email), :should + q2.add_query Ferret::Search::TermQuery.new("to", p.email), :should + end + query.add_query q2, :must end - query += " -label:spam" unless opts[:load_spam] || opts[:labels] == :spam || - (opts[:labels] && opts[:labels].include?(:spam)) - query += " -label:killed" unless opts[:load_killed] || opts[:labels] == :killed || - (opts[:labels] && opts[:labels].include?(:killed)) + query.add_query Ferret::Search::TermQuery.new("label", "spam"), :must_not unless opts[:load_spam] || labels.include?(:spam) + query.add_query Ferret::Search::TermQuery.new("label", "deleted"), :must_not unless opts[:load_deleted] || labels.include?(:deleted) + query.add_query Ferret::Search::TermQuery.new("label", "killed"), :must_not unless opts[:load_killed] || labels.include?(:killed) query end - def load_sources fn=Redwood::SOURCE_FN - @sources = Hash[*(Redwood::load_yaml_obj(fn) || []).map { |s| [s.id, s] }.flatten] - @sources_dirty = false - end - def save_sources fn=Redwood::SOURCE_FN if @sources_dirty || @sources.any? { |id, s| s.dirty? } bakfn = fn + ".bak" @@ -335,7 +308,7 @@ protected File.chmod 0600, fn FileUtils.mv fn, bakfn, :force => true unless File.exists?(bakfn) && File.size(bakfn) > File.size(fn) end - Redwood::save_yaml_obj @sources.values, fn + Redwood::save_yaml_obj @sources.values, fn File.chmod 0600, fn end @sources_dirty = false