removed cruft from index.rb, and made source_info an indexed field, such that

sup-import --rebuild --start_at will be able to find the right messages to
delete.

git-svn-id: svn://rubyforge.org/var/svn/sup/trunk@61 5c8cc53c-5e98-4d25-b20a-d8db53a31250

diff --git a/bin/sup-import b/bin/sup-import
index c1baf0485697d31cfca62ad6956790238c5eb22e..f040ef91e1efeb564a400df52c8931a7a89c38d8 100644 (file)
--- a/bin/sup-import
+++ b/bin/sup-import
@@ -88,11 +88,22 @@ puts "loaded index of #{index.size} messages"
 sources = ARGV.map do |fn|
   source = index.source_for fn
   unless source
-    source = Redwood::MBox::Loader.new(fn, 0, !unusual, !!archive)
+    source = 
+      case fn
+      when %r!^imaps?://!
+        print "Username for #{fn}: "
+        username = $stdin.gets.chomp
+        print "Password for #{fn} (warning: cleartext): "
+        password = $stdin.gets.chomp
+        Redwood::IMAP.new(fn, username, password, nil, !unusual, !!archive)
+      else
+        Redwood::MBox::Loader.new(fn, nil, !unusual, !!archive)
+      end
     index.add_source source
   end
   source
 end
+
 sources = (sources + index.usual_sources).uniq if the_usual
 if rebuild || force_rebuild
   if start_at
@@ -122,8 +133,8 @@ begin
         else
           found[m.id] = true
         end
-
-        m.remove_label :unread if m.mbox_status == "RO" unless force_read
+        m.remove_label :unread if m.status == "RO" unless force_read
+        puts "# message at #{offset} labels #{labels.inspect}"
         if (rebuild || force_rebuild) && 
             (docid, entry = index.load_entry_for_id(m.id)) && entry
           if force_rebuild || entry[:source_info].to_i != offset
@@ -152,16 +163,12 @@ end
 
 if rebuild || force_rebuild
   puts "deleting missing messages from the index..."
-  if start_at
-    $stderr.puts "deleting currently don't work with --start-at. sorry!"
-    break
-  end
   numdel = num = 0
   sources.each do |source|
     raise "no source id for #{source}" unless source.id
     q = "+source_id:#{source.id}"
     q += " +source_info: >= #{start_at}" if start_at
-    p q
+    #p q
     num += index.index.search_each(q, :limit => :all) do |docid, score|
       mid = index.index[docid][:message_id]
       next if found[mid]

diff --git a/lib/sup/index.rb b/lib/sup/index.rb
index 90bfc2836067ff86d7d3352a6c35d5e847fbfb24..52d099efef17652908e12dc88f85dc262b9528d4 100644 (file)
--- a/lib/sup/index.rb
+++ b/lib/sup/index.rb
@@ -18,17 +18,11 @@ end
 class Index
   include Singleton
 
-  LOAD_THREAD_PETIT_DELAY = 0.1
-  LOAD_THREAD_GRAND_DELAY = 5
-
-  MESSAGES_AT_A_TIME = 10
-
   attr_reader :index # debugging only
   
   def initialize dir=BASE_DIR
     @dir = dir
     @mutex = Mutex.new
-    @load_thread = nil # loads new messages
     @sources = {}
     @sources_dirty = false
 
@@ -71,7 +65,7 @@ class Index
       field_infos = Ferret::Index::FieldInfos.new :store => :yes
       field_infos.add_field :message_id
       field_infos.add_field :source_id
-      field_infos.add_field :source_info, :index => :no, :term_vector => :no
+      field_infos.add_field :source_info
       field_infos.add_field :date, :index => :untokenized
       field_infos.add_field :body, :store => :no
       field_infos.add_field :label
@@ -190,30 +184,24 @@ class Index
     source = @sources[doc[:source_id].to_i]
     #puts "building message #{doc[:message_id]} (#{source}##{doc[:source_info]})"
     raise "invalid source #{doc[:source_id]}" unless source
+    raise "no snippet" unless doc[:snippet]
+
     begin
-      raise "no snippet" unless doc[:snippet]
       Message.new source, doc[:source_info].to_i, 
                   doc[:label].split(" ").map { |s| s.intern },
                   doc[:snippet]
     rescue MessageFormatError => e
       raise IndexError.new(source, "error building message #{doc[:message_id]} at #{source}/#{doc[:source_info]}: #{e.message}")
-      nil
-    end
-  end
-
-  def start_load_thread
-    return if @load_thread
-    @load_thread = true
-    @load_thread = ::Thread.new do
-      while @load_thread
-        load_some_entries ENTRIES_AT_A_TIME, LOAD_THREAD_PETIT_DELAY, LOAD_THREAD_GRAND_DELAY
-      end
+#     rescue StandardError => e
+#       Message.new_from_index doc, <<EOS
+# An error occurred while loading this message. It is possible that the source
+# has changed, or (in the case of remote sources) is down. The error was:
+# #{e.message}
+# EOS
     end
   end
 
-  def end_load_thread; @load_thread = nil; end
   def fresh_thread_id; @next_thread_id += 1; end
-
   def wrap_subj subj; "__START_SUBJECT__ #{subj} __END_SUBJECT__"; end
 
   def add_message m
@@ -321,33 +309,6 @@ protected
     end
     @sources_dirty = false
   end
-
-  def load_some_entries max=ENTRIES_AT_A_TIME, delay1=nil, delay2=nil
-    num = 0
-    begin
-      @sources.each_with_index do |source, source_id|
-        next if source.done? || num >= max
-        source.each do |source_info, label|
-          begin
-            m = Message.new(source, source_info, label + [:inbox])
-            add_message m unless contains_id? m.id
-            puts m.content.inspect
-            num += 1
-          rescue MessageFormatError => e
-            $stderr.puts "ignoring erroneous message at #{source}##{source_info}: #{e.message}"
-          end
-          break if num >= max
-          sleep delay1 if delay1
-        end
-        Redwood::log "loaded #{num} entries from #{source}"
-        sleep delay2 if delay2
-      end
-    ensure
-      save_sources
-      save_index
-    end
-    num
-  end
 end
 
 end