X-Git-Url: https://git.cworth.org/git?a=blobdiff_plain;f=lib%2Fsup%2Findex.rb;h=2e5ba49872481cd46ddb5322f16b870fe7bf172c;hb=3a70f7918c6385fa2e3b74d2b4f6eca258af107b;hp=52d099efef17652908e12dc88f85dc262b9528d4;hpb=4b8a07b98ce84f2d56a3b4ddcbbefc12e9da8b36;p=sup diff --git a/lib/sup/index.rb b/lib/sup/index.rb index 52d099e..2e5ba49 100644 --- a/lib/sup/index.rb +++ b/lib/sup/index.rb @@ -2,30 +2,26 @@ require 'thread' require 'fileutils' -require_gem 'ferret', ">= 0.10.13" +require 'ferret' +#require_gem 'ferret', ">= 0.10.13" module Redwood -class IndexError < StandardError - attr_reader :source - - def initialize source, s - super s - @source = source - end -end - class Index include Singleton - attr_reader :index # debugging only - + attr_reader :index def initialize dir=BASE_DIR @dir = dir - @mutex = Mutex.new @sources = {} @sources_dirty = false + wsa = Ferret::Analysis::WhiteSpaceAnalyzer.new false + sa = Ferret::Analysis::StandardAnalyzer.new Ferret::Analysis::FULL_ENGLISH_STOP_WORDS, true + @analyzer = Ferret::Analysis::PerFieldAnalyzer.new wsa + @analyzer[:body] = sa + @qparser ||= Ferret::QueryParser.new :default_field => :body, :analyzer => @analyzer + self.class.i_am_the_instance self end @@ -44,24 +40,22 @@ class Index raise "duplicate source!" if @sources.include? source @sources_dirty = true source.id ||= @sources.size - source.id += 1 while @sources.member? source.id + ##TODO: why was this necessary? + ##source.id += 1 while @sources.member? source.id @sources[source.id] = source end def source_for name; @sources.values.find { |s| s.is_source_for? name }; end def usual_sources; @sources.values.find_all { |s| s.usual? }; end + def sources; @sources.values; end def load_index dir=File.join(@dir, "ferret") - wsa = Ferret::Analysis::WhiteSpaceAnalyzer.new false - sa = Ferret::Analysis::StandardAnalyzer.new - analyzer = Ferret::Analysis::PerFieldAnalyzer.new wsa - analyzer[:body] = sa - if File.exists? dir - Redwood::log "loading index" - @index = Ferret::Index::Index.new(:path => dir, :analyzer => analyzer) + Redwood::log "loading index..." + @index = Ferret::Index::Index.new(:path => dir, :analyzer => @analyzer) + Redwood::log "loaded index of #{@index.size} messages" else - Redwood::log "creating index" + Redwood::log "creating index..." field_infos = Ferret::Index::FieldInfos.new :store => :yes field_infos.add_field :message_id field_infos.add_field :source_id @@ -75,7 +69,7 @@ class Index field_infos.add_field :refs field_infos.add_field :snippet, :index => :no, :term_vector => :no field_infos.create_index dir - @index = Ferret::Index::Index.new(:path => dir, :analyzer => analyzer) + @index = Ferret::Index::Index.new(:path => dir, :analyzer => @analyzer) end end @@ -86,7 +80,9 @@ class Index source ||= entry[:source_id].to_i source_info ||= entry[:source_info].to_i end - raise "no entry and no source info for message #{m.id}" unless source && source_info + + ## this happens sometimes. i'm not sure why. ferret bug? + raise "no entry and no source info for message #{m.id}: source #{source.inspect}, info #{source_info.inspect}, entry #{entry.inspect}, query #{Ferret::Search::TermQuery.new(:message_id, m.id)}, results #{@index.search(Ferret::Search::TermQuery.new(:message_id, m.id)).inspect}" unless source && source_info raise "deleting non-corresponding entry #{docid}" unless @index[docid][:message_id] == m.id @index.delete docid @@ -108,7 +104,7 @@ class Index ## in scotland, frikkin' huuuge. EACH_BY_DATE_NUM = 100 def each_id_by_date opts={} - return if @index.size == 0 # otherwise ferret barfs + return if @index.size == 0 # otherwise ferret barfs ###TODO: remove this once my ferret patch is accepted query = build_query opts offset = 0 while true @@ -121,12 +117,15 @@ class Index end def num_results_for opts={} - query = build_query opts - x = @index.search(query).total_hits - Redwood::log "num_results_for: have #{x} for query #{query}" - x + return 0 if @index.size == 0 # otherwise ferret barfs ###TODO: remove this once my ferret patch is accepted + q = build_query opts + index.search(q).total_hits end + ## yield all messages in the thread containing 'm' by repeatedly + ## querying the index. yields pairs of message ids and + ## message-building lambdas, so that building an unwanted message + ## can be skipped in the block if desired. SAME_SUBJECT_DATE_LIMIT = 7 def each_message_in_thread_for m, opts={} messages = {} @@ -184,25 +183,24 @@ class Index source = @sources[doc[:source_id].to_i] #puts "building message #{doc[:message_id]} (#{source}##{doc[:source_info]})" raise "invalid source #{doc[:source_id]}" unless source - raise "no snippet" unless doc[:snippet] - - begin - Message.new source, doc[:source_info].to_i, - doc[:label].split(" ").map { |s| s.intern }, - doc[:snippet] - rescue MessageFormatError => e - raise IndexError.new(source, "error building message #{doc[:message_id]} at #{source}/#{doc[:source_info]}: #{e.message}") -# rescue StandardError => e -# Message.new_from_index doc, < Time.at(doc[:date].to_i), + "subject" => unwrap_subj(doc[:subject]), + "from" => doc[:from], + "to" => doc[:to], + "message-id" => doc[:message_id], + "references" => doc[:refs], + } + + Message.new :source => source, :source_info => doc[:source_info].to_i, + :labels => doc[:label].split(" ").map { |s| s.intern }, + :snippet => doc[:snippet], :header => fake_header end def fresh_thread_id; @next_thread_id += 1; end def wrap_subj subj; "__START_SUBJECT__ #{subj} __END_SUBJECT__"; end + def unwrap_subj subj; subj =~ /__START_SUBJECT__ (.*?) __END_SUBJECT__/ && $1; end def add_message m return false if contains? m @@ -211,7 +209,7 @@ class Index if m.source.is_a? Integer m.source else - m.source.id or raise "unregistered source #{m.source}" + m.source.id or raise "unregistered source #{m.source} (id #{m.source.id.inspect})" end to = (m.to + m.cc + m.bcc).map { |x| x.email }.join(" ") @@ -279,21 +277,23 @@ class Index protected - ## TODO: convert this to query objects rather than strings + def parse_user_query_string str; @qparser.parse str; end def build_query opts - query = "" - query += opts[:labels].map { |t| "+label:#{t}" }.join(" ") if opts[:labels] - query += " +label:#{opts[:label]}" if opts[:label] - query += " #{opts[:content]}" if opts[:content] + query = Ferret::Search::BooleanQuery.new + query.add_query opts[:qobj], :must if opts[:qobj] + labels = ([opts[:label]] + (opts[:labels] || [])).compact + labels.each { |t| query.add_query Ferret::Search::TermQuery.new("label", t.to_s), :must } if opts[:participants] - query += "+(" + - opts[:participants].map { |p| "from:#{p.email} OR to:#{p.email}" }.join(" OR ") + ")" + q2 = Ferret::Search::BooleanQuery.new + opts[:participants].each do |p| + q2.add_query Ferret::Search::TermQuery.new("from", p.email), :should + q2.add_query Ferret::Search::TermQuery.new("to", p.email), :should + end + query.add_query q2, :must end - query += " -label:spam" unless opts[:load_spam] || opts[:labels] == :spam || - (opts[:labels] && opts[:labels].include?(:spam)) - query += " -label:killed" unless opts[:load_killed] || opts[:labels] == :killed || - (opts[:labels] && opts[:labels].include?(:killed)) + query.add_query Ferret::Search::TermQuery.new("label", "spam"), :must_not unless opts[:load_spam] || labels.include?(:spam) + query.add_query Ferret::Search::TermQuery.new("label", "killed"), :must_not unless opts[:load_killed] || labels.include?(:killed) query end @@ -304,8 +304,13 @@ protected def save_sources fn=Redwood::SOURCE_FN if @sources_dirty || @sources.any? { |id, s| s.dirty? } - FileUtils.mv fn, fn + ".bak", :force => true if File.exists? fn - Redwood::save_yaml_obj @sources.values, fn + bakfn = fn + ".bak" + if File.exists? fn + File.chmod 0600, fn + FileUtils.mv fn, bakfn, :force => true unless File.exists?(bakfn) && File.size(bakfn) > File.size(fn) + end + Redwood::save_yaml_obj @sources.values, fn + File.chmod 0600, fn end @sources_dirty = false end