From: wmorgan <wmorgan@5c8cc53c-5e98-4d25-b20a-d8db53a31250>
Date: Sun, 1 Apr 2007 02:28:01 +0000 (+0000)
Subject: moved sup-import to sup-sync and changed interface a lot. note that trollop 1.5 is... 
X-Git-Url: https://git.cworth.org/git?a=commitdiff_plain;h=8401bef72bf71646259cb5a2c2d50c301ac2f2b4;p=sup

moved sup-import to sup-sync and changed interface a lot. note that trollop 1.5 is now required

git-svn-id: svn://rubyforge.org/var/svn/sup/trunk@353 5c8cc53c-5e98-4d25-b20a-d8db53a31250
---

diff --git a/Manifest.txt b/Manifest.txt
index 0f33838..a1252e1 100644
--- a/Manifest.txt
+++ b/Manifest.txt
@@ -6,7 +6,7 @@ README.txt
 Rakefile
 bin/sup
 bin/sup-add
-bin/sup-import
+bin/sup-sync
 bin/sup-recover-sources
 doc/FAQ.txt
 doc/Philosophy.txt
diff --git a/Rakefile b/Rakefile
index 22f238d..955dee4 100644
--- a/Rakefile
+++ b/Rakefile
@@ -16,7 +16,7 @@ Hoe.new('sup', Redwood::VERSION) do |p|
   p.url = p.paragraphs_of('README.txt', 0).first.split(/\n/)[2].gsub(/^\s+/, "")
   p.changes = p.paragraphs_of('History.txt', 0..0).join("\n\n")
   p.email = "wmorgan-sup@masanjin.net"
-  p.extra_deps = [['ferret', '>= 0.10.13'], ['ncurses', '>= 0.9.1'], ['rmail', '>= 0.17'], 'highline', 'net-ssh', 'trollop']
+  p.extra_deps = [['ferret', '>= 0.10.13'], ['ncurses', '>= 0.9.1'], ['rmail', '>= 0.17'], 'highline', 'net-ssh', ['trollop', '>= 1.5']]
 end
 
 rule 'ss?.png' => 'ss?-small.png' do |t|
diff --git a/bin/sup-import b/bin/sup-import
deleted file mode 100644
index a1eadfa..0000000
--- a/bin/sup-import
+++ /dev/null
@@ -1,157 +0,0 @@
-#!/usr/bin/env ruby
-
-require 'uri'
-require 'rubygems'
-require 'trollop'
-require "sup"
-
-class Float
-  def to_s; sprintf '%.2f', self; end
-end
-
-class Numeric
-  def to_time_s
-    i = to_i
-    sprintf "%d:%02d:%02d", i / 3600, (i / 60) % 60, i % 60
-  end
-end
-
-def time
-  startt = Time.now
-  yield
-  Time.now - startt
-end
-
-opts = Trollop::options do
-  version "sup-import (sup #{Redwood::VERSION})"
-  banner <<EOS
-Imports messages into the Sup index from one or more sources.
-
-Usage:
-  sup-import [options] <source>*
-
-where <source>* is zero or more source URIs or mbox filenames. If no
-sources are given, imports messages from all sources marked as
-"usual".
-
-Options are:
-EOS
-  opt :archive, "Automatically archive any imported messages."
-  opt :read, "Automatically mark as read any imported messages."
-  opt :verbose, "Print message ids as they're processed."
-  opt :optimize, "As the last stage of the import, optimize the index."
-  text <<EOS
-
-The following options allow sup-import to consider *all* messages in the
-source, not just new ones:
-EOS
-  opt :rebuild, "Scan over the entire source and update the index to account for any messages that have been deleted, altered, or moved from another source."
-  opt :full_rebuild, "Re-insert all messages in the source, not just ones that have changed or are new."
-  opt :start_at, "For rescan and rebuild, start at the given offset.", :type => :int
-  opt :overwrite_state, "For --full-rebuild, overwrite the message state to the default state for that source, obeying --archive and --read if given."
-end
-Trollop::die :start_at, "must be non-negative" if (opts[:start_at] || 0) < 0
-Trollop::die :start_at, "requires either --rebuild or --full-rebuild" if opts[:start_at] && !(opts[:rebuild] || opts[:full_rebuild])
-Trollop::die :overwrite_state, "requires --full-rebuild" if opts[:overwrite_state] && !opts[:full_rebuild]
-Trollop::die :force_rebuild, "cannot be specified with --rebuild" if opts[:full_rebuild] && opts[:rebuild]
-
-Redwood::start
-index = Redwood::Index.new
-index.load
-
-sources = ARGV.map do |uri|
-  uri = "mbox://#{uri}" unless uri =~ %r!://!
-  index.source_for uri or Trollop::die "Unknown source: #{uri}. Did you add it with sup-add first?"
-end
-
-sources = index.usual_sources if sources.empty?
-
-if opts[:rebuild] || opts[:full_rebuild]
-  if opts[:start_at]
-    sources.each { |s| s.seek_to! opts[:start_at] }
-  else
-    sources.each { |s| s.reset! }
-  end
-end
-
-last_update = start = Time.now
-found = {}
-begin
-  sources.each do |source|
-    num_added = 0
-    num_updated = 0
-    puts "Scanning #{source}..."
-    Redwood::PollManager.add_new_messages_from source do |m, offset, entry|
-      ## if the entry exists on disk
-      if entry && !opts[:overwrite_state]
-        m.labels = entry[:label].split(/\s+/).map { |x| x.intern }
-      else
-        ## m.labels defaults to labels from the source
-        m.labels -= [:inbox] if opts[:archive]
-        m.labels -= [:unread] if opts[:read]
-      end
-
-      if Time.now - last_update > 60
-        last_update = Time.now
-        elapsed = last_update - start
-        pctdone = source.respond_to?(:pct_done) ? source.pct_done : 100.0 * (source.cur_offset.to_f - source.start_offset).to_f / (source.end_offset - source.start_offset).to_f
-        remaining = (100.0 - pctdone) * (elapsed.to_f / pctdone)
-        puts "## #{num_added + num_updated} (#{pctdone}% done) read; #{elapsed.to_time_s} elapsed; est. #{remaining.to_time_s} remaining"
-      end
-
-      ## update if...
-      if entry.nil? # it's a new message; or
-        puts "Adding message at #{offset}, labels: #{m.labels * ' '}" if opts[:verbose]
-        num_added += 1
-        found[m.id] = true
-        m
-      elsif opts[:full_rebuild] || # we're updating everyone; or
-          (opts[:rebuild] && (entry[:source_id].to_i != source.id || entry[:source_info].to_i != offset)) # we're updating just the changed ones
-        puts "Updating message at #{offset} (from #{m.from.longname}, subject '#{m.subj}'), source #{entry[:source_id]} => #{source.id}, offset #{entry[:source_info]} => #{offset}, labels: {#{m.labels * ', '}}" if opts[:verbose]
-        num_updated += 1 unless found[m.id]
-        found[m.id] = true
-        m
-      else
-        found[m.id] = true
-        nil
-      end
-    end
-    puts "Added #{num_added}, updated #{num_updated} messages from #{source}."
-  end
-ensure
-  puts "Saving index and sources..."
-  index.save
-  Redwood::finish
-end
-
-## delete any messages in the index that claim they're from one of
-## these sources, but that we didn't see.
-##
-## kinda crappy code here, because we delve directly into the Ferret
-## API.
-##
-## TODO: move this to Index, i suppose.
-if opts[:rebuild] || opts[:full_rebuild]
-  puts "Deleting missing messages from the index..."
-  numdel = num = 0
-  sources.each do |source|
-    raise "no source id for #{source}" unless source.id
-    q = "+source_id:#{source.id}"
-    q += " +source_info: >= #{opts[:start_at]}" if opts[:start_at]
-    num += index.index.search_each(q, :limit => :all) do |docid, score|
-      mid = index.index[docid][:message_id]
-#      puts "got #{mid}"
-      next if found[mid]
-      puts "Deleting #{mid}" if opts[:verbose]
-      index.index.delete docid
-      numdel += 1
-    end
-  end
-  puts "Deleted #{numdel} / #{num} messages"
-end
-
-if opts[:optimize]
-  puts "Optimizing index..."
-  optt = time { index.index.optimize }
-  puts "Optimized index of size #{index.size} in #{optt}s."
-end
diff --git a/bin/sup-sync b/bin/sup-sync
new file mode 100644
index 0000000..68f462d
--- /dev/null
+++ b/bin/sup-sync
@@ -0,0 +1,235 @@
+#!/usr/bin/env ruby
+
+require 'uri'
+require 'rubygems'
+require 'trollop'
+require "sup"
+
+class Float
+  def to_s; sprintf '%.2f', self; end
+end
+
+class Numeric
+  def to_time_s
+    i = to_i
+    sprintf "%d:%02d:%02d", i / 3600, (i / 60) % 60, i % 60
+  end
+end
+
+def time
+  startt = Time.now
+  yield
+  Time.now - startt
+end
+
+opts = Trollop::options do
+  version "sup-sync (sup #{Redwood::VERSION})"
+  banner <<EOS
+Synchronizes the Sup index with one or more message sources by adding
+messages, deleting messages, or changing message state in the index as
+appropriate.
+
+"Message state" means read/unread, archived/inbox, starred/unstarred,
+and all user-defined labels on each message.
+
+"Default source state" refers to any state that a source itself has
+keeps about a message. Sup-sync uses this information when adding a
+new message to the index. The source state is typically limited to
+read/unread, archived/inbox status and a single label based on the
+source name. Messages using the default source state are placed in
+the inbox (i.e. not archived) and unstarred.
+
+Usage:
+  sup-sync [options] <source>*
+
+where <source>* is zero or more source URIs. If no sources are given,
+sync from all usual sources.
+
+Supported source URIs:
+  mbox://<path to mbox file>,      e.g. mbox:///var/spool/mail/me
+  maildir://<path to maildir dir>, e.g. maildir:///home/me/Maildir
+  mbox+ssh://<machine>/<path to mbox file>
+  imap://<machine>/[<folder>]
+  imaps://<machine>/[<folder>]
+
+Options controlling WHICH messages sup-sync operates on:
+EOS
+  opt :new, "Operate on new messages only. Don't scan over the entire source. (Default.)", :short => :none
+  opt :changed, "Scan over the entire source for messages that have been deleted, altered, or moved from another source. (In the case of mbox sources, this includes all messages AFTER an altered message.)"
+  opt :restored, "Operate only on those messages included in a dump file as specified by --restore."
+  opt :all, "Operate on all messages in the source, regardless of newness or changedness."
+  opt :start_at, "For --changed and --all, start at a particular offset.", :type => :int
+
+text <<EOS
+
+Options controlling HOW message state is altered:
+EOS
+  opt :asis, "If the message is already in the index, preserve its state. Otherwise, use default source state. (Default.)", :short => :none
+  opt :restore, "Restore message state from a dump file created with sup-dump. If a message is not in this dumpfile, act as --asis.", :type => String, :short => :none
+  opt :discard, "Discard any message state in the index and use the default source state. Dangerous!", :short => :none
+  opt :archive, "When using the default source state, mark messages as archived.", :short => "-x"
+  opt :read, "When using the default source state, mark messages as read."
+  opt :extra_labels, "When using the default source state, also apply these user-defined labels. Should be a comma-separated list.", :type => String, :short => :none
+
+text <<EOS
+
+Other options:
+EOS
+  opt :verbose, "Print message ids as they're processed."
+  opt :optimize, "As the final operation, optimize the index."
+  opt :dry_run, "Don't actually modify the index. Probably only useful with --verbose.", :short => "-n"
+  opt :version, "Show version information", :short => :none
+
+  conflicts :changed, :all, :new, :restored
+  conflicts :asis, :restore, :discard
+end
+Trollop::die :restored, "requires --restore" if opts[:restore] unless opts[:restored]
+if opts[:start_at]
+  Trollop::die :start_at, "must be non-negative" if opts[:start_at] < 0
+  Trollop::die :start_at, "requires either --changed or --all" unless opts[:changed] || opts[:all]
+end
+
+target = [:new, :changed, :all, :restored].find { |x| opts[x] } || :new
+op = [:asis, :restore, :discard].find { |x| opts[x] } || :asis
+
+Redwood::start
+index = Redwood::Index.new
+index.load
+
+restored_state =
+  if opts[:restore]
+    dump = {}
+    $stderr.puts "Loading state dump from #{opts[:restore]}..."
+    IO.foreach opts[:restore] do |l|
+      l =~ /^(\S+) (\d+) (\d+) \((.*?)\)$/ or raise "Can't read dump line: #{l.inspect}"
+      mid, source_id, source_info, labels = $1, $2.to_i, $3.to_i, $4
+      dump[mid] = labels.split(" ").map { |x| x.intern }
+    end
+    $stderr.puts "Read #{dump.size} entries from dump file."
+    dump
+  else
+    {}
+  end
+
+sources = ARGV.map do |uri|
+  uri = "mbox://#{uri}" unless uri =~ %r!://!
+  index.source_for uri or Trollop::die "Unknown source: #{uri}. Did you add it with sup-add first?"
+end
+
+sources = index.usual_sources if sources.empty?
+
+unless target == :new
+  if opts[:start_at]
+    sources.each { |s| s.seek_to! opts[:start_at] }
+  else
+    sources.each { |s| s.reset! }
+  end
+end
+
+last_info_time = start_time = Time.now
+seen = {}
+begin
+  sources.each do |source|
+    num_added, num_updated, num_scanned = 0, 0, 0
+    $stderr.puts "Scanning #{source}..."
+
+    Redwood::PollManager.add_messages_from source do |m, offset, entry|
+      num_scanned += 1
+      seen[m.id] = true
+
+      ## skip if we're operating only on changed messages, the message
+      ## is in the index, and it's unchanged from what the source is
+      ## reporting.
+      next if target == :changed && entry && entry[:source_id].to_i == source.id && entry[:source_info].to_i == offset
+
+      ## skip if we're operating on restored messages, and this one
+      ## ain't.
+      next if target == :restored && !restored_state[m.id]
+
+      ## m.labels is the default source labels. tweak these according
+      ## to default source state modification flags.
+      m.labels -= [:inbox] if opts[:archive]
+      m.labels -= [:unread] if opts[:read]
+      m.labels += opts[:extra_labels].split(/\s*,\s*/).map { |x| x.intern } if opts[:extra_labels]
+
+      ## get the state currently in the index
+      index_state =
+        if entry
+          entry[:label].split(/\s+/).map { |x| x.intern }
+        else
+          nil
+        end
+
+      ## assign message labels based on the operation we're performing
+      case op
+      when :asis
+        m.labels = index_state if index_state
+      when :restore
+        ## if the entry exists on disk
+        if restored_state[m.id]
+          m.labels = restored_state[m.id]
+        elsif index_state
+          m.labels = index_state
+        end
+      when :discard
+        ## nothin! use default source labels
+      end
+
+      if Time.now - last_info_time > 60
+        last_info_time = Time.now
+        elapsed = last_info_time - start_time
+        pctdone = source.respond_to?(:pct_done) ? source.pct_done : 100.0 * (source.cur_offset.to_f - source.start_time_offset).to_f / (source.end_offset - source.start_time_offset).to_f
+        remaining = (100.0 - pctdone) * (elapsed.to_f / pctdone)
+        puts "## #{num_added + num_updated} (#{pctdone}% done) read; #{elapsed.to_time_s} elapsed; est. #{remaining.to_time_s} remaining"
+      end
+
+      if index_state.nil?
+        puts "Adding message #{source}##{offset} with state {#{m.labels * ', '}}" if opts[:verbose]
+        num_added += 1
+      else
+        puts "Updating message #{source}##{offset}, source #{entry[:source_id]} => #{source.id}, offset #{entry[:source_info]} => #{offset}, state {#{index_state * ', '}} => {#{m.labels * ', '}}" if opts[:verbose]
+        num_updated += 1
+      end
+
+      opts[:dry_run] ? nil : m
+    end
+    $stderr.puts "Added #{num_added}, updated #{num_updated} messages from #{source}."
+  end
+ensure
+  $stderr.puts "Saving index and sources..."
+  index.save
+  Redwood::finish
+end
+
+## delete any messages in the index that claim they're from one of
+## these sources, but that we didn't see.
+##
+## kinda crappy code here, because we delve directly into the Ferret
+## API.
+##
+## TODO: move this to Index, i suppose.
+if target == :all || target == :changed
+  $stderr.puts "Deleting missing messages from the index..."
+  num_del, num_scanned = 0, 0
+  sources.each do |source|
+    raise "no source id for #{source}" unless source.id
+    q = "+source_id:#{source.id}"
+    q += " +source_info: >= #{opts[:start_at]}" if opts[:start_at]
+    index.index.search_each(q, :limit => :all) do |docid, score|
+      num_scanned += 1
+      mid = index.index[docid][:message_id]
+      unless seen[mid]
+        puts "Deleting #{mid}" if opts[:verbose]
+        index.index.delete docid unless opts[:dry_run]
+        num_del += 1
+      end
+    end
+  end
+  $stderr.puts "Deleted #{num_del} / #{num_scanned} messages"
+end
+
+if opts[:optimize]
+  $stderr.puts "Optimizing index..."
+  optt = time { index.index.optimize unless opts[:dry_run] }
+  $stderr.puts "Optimized index of size #{index.size} in #{optt}s."
+end
diff --git a/doc/TODO b/doc/TODO
index 7a78fb0..0876942 100644
--- a/doc/TODO
+++ b/doc/TODO
@@ -3,24 +3,24 @@ for 0.0.8
 _ split out threading & message chunk parsing to a separate library
 _ ferret upgrade script (dump & restore)
 _ nice little startup config program
-x maildir
-x bugfix: single-line messages come empty upon reply
 _ bugfix: when one new message comes into an imap folder, we don't
    catch it until a reload
 _ bugfix: triggering a pageup when cursor scrolling up jumps to the
    bottom of the page rather than the next line
-x compose in thread-view-mode auto-fills in person
 _ bugfix: stars on messages with blue backgrounds still have green bgs
-x bugfix: mark messages as read immediately when t-v-m is opened
 _ bugfix: m in thread-view-mode when a person is not selected should open up a
   blank compose-mode rather than do nothing
 _ Net::SMTP support (cuz I'm going to need it soon)
-x bugfix: 'N' in thread-view-mode (expand only new messages) crashes
-_ bugfix: detect source corruption at startup
 _ bugfix: add new message counts until keypress
 _ bugfix: attachment filenames sometimes not detected (filename=)
 _ bugfix: final logging messages to stdout?
 _ bugfix: mbox directory shouldn't generate an exception, just an error
+x bugfix: mark messages as read immediately when t-v-m is opened
+x compose in thread-view-mode auto-fills in person
+x bugfix: 'N' in thread-view-mode (expand only new messages) crashes
+x bugfix: detect source corruption at startup
+x maildir
+x bugfix: single-line messages come empty upon reply
 
 for 0.0.9
 ---------
@@ -30,6 +30,7 @@ _ select all, starred, to me, etc
 _ undo
 _ gmail
 _ warnings: top-posting, missing attachment, ruby-talk:XXXX detection
+_ mboxz (compressed mbox)
 
 future
 ------
diff --git a/lib/sup.rb b/lib/sup.rb
index f9021ba..a98ee04 100644
--- a/lib/sup.rb
+++ b/lib/sup.rb
@@ -97,6 +97,7 @@ module Redwood
 
   ## not really a good place for this, so I'll just dump it here.
   def report_broken_sources
+    return unless BufferManager.instantiated?
     broken_sources = Index.usual_sources.select { |s| s.broken? }
     unless broken_sources.empty?
       BufferManager.spawn "Broken source report", TextMode.new(<<EOM)
diff --git a/lib/sup/draft.rb b/lib/sup/draft.rb
index e234c10..4beaf3e 100644
--- a/lib/sup/draft.rb
+++ b/lib/sup/draft.rb
@@ -22,7 +22,7 @@ class DraftManager
     my_message = nil
     @source.each do |thisoffset, theselabels|
       m = Message.new :source => @source, :source_info => thisoffset, :labels => theselabels
-      Index.add_message m
+      Index.sync_message m
       UpdateManager.relay self, :add, m
       my_message = m if thisoffset == offset
     end
diff --git a/lib/sup/imap.rb b/lib/sup/imap.rb
index 6abe536..f1a9e43 100644
--- a/lib/sup/imap.rb
+++ b/lib/sup/imap.rb
@@ -208,7 +208,7 @@ private
         e
       end
 
-    message += " It is likely that messages have been deleted from this IMAP mailbox. Please run sup-import --rebuild #{to_s} to correct this problem." if opts[:suggest_rebuild]
+    message += " It is likely that messages have been deleted from this IMAP mailbox. Please run sup-sync --changed #{to_s} to correct this problem." if opts[:suggest_rebuild]
 
     self.broken_msg = message
     Redwood::log message
diff --git a/lib/sup/index.rb b/lib/sup/index.rb
index 28fde2a..5d1a7b3 100644
--- a/lib/sup/index.rb
+++ b/lib/sup/index.rb
@@ -72,25 +72,46 @@ class Index
     end
   end
 
-  ## Update the message state on disk, by deleting and re-adding it.
-  ## The message must exist in the index. docid and entry are found
-  ## unless given.
+  ## Syncs the message to the index: deleting if it's already there,
+  ## and adding either way. Index state will be determined by m.labels.
   ##
-  ## Overwrites the labels on disk with the new labels in 'm', so that
-  ## we can actually change message state.
-  def update_message m, docid=nil, entry=nil
-    unless docid && entry
-      docid, entry = load_entry_for_id m.id
-      raise ArgumentError, "cannot find #{m.id} in the index" unless entry
-    end
+  ## docid and entry can be specified if they're already known.
+  def sync_message m, docid=nil, entry=nil
+    docid, entry = load_entry_for_id m.id unless docid && entry
 
-    raise "no entry and no source info for message #{m.id}" unless m.source && m.source_info
+    raise "no source info for message #{m.id}" unless m.source && m.source_info
+    raise "trying deleting non-corresponding entry #{docid}" if docid && @index[docid][:message_id] != m.id
 
-    raise "deleting non-corresponding entry #{docid}" unless @index[docid][:message_id] == m.id
+    source_id = 
+      if m.source.is_a? Integer
+        raise "Debugging: integer source set"
+        m.source
+      else
+        m.source.id or raise "unregistered source #{m.source} (id #{m.source.id.inspect})"
+      end
 
-    @index.delete docid
-    add_message m
+    to = (m.to + m.cc + m.bcc).map { |x| x.email }.join(" ")
+    d = {
+      :message_id => m.id,
+      :source_id => source_id,
+      :source_info => m.source_info,
+      :date => m.date.to_indexable_s,
+      :body => m.content,
+      :snippet => m.snippet,
+      :label => m.labels.join(" "),
+      :from => m.from ? m.from.email : "",
+      :to => (m.to + m.cc + m.bcc).map { |x| x.email }.join(" "),
+      :subject => wrap_subj(Message.normalize_subj(m.subj)),
+      :refs => (m.refs + m.replytos).uniq.join(" "),
+    }
+
+    @index.delete docid if docid
+    @index.add_document d
+    
     docid, entry = load_entry_for_id m.id
+    ## this hasn't been triggered in a long time. TODO: decide whether it's still a problem.
+    raise "just added message #{m.id} but couldn't find it in a search" unless docid
+    true
   end
 
   def save_index fn=File.join(@dir, "ferret")
@@ -210,41 +231,6 @@ class Index
   def wrap_subj subj; "__START_SUBJECT__ #{subj} __END_SUBJECT__"; end
   def unwrap_subj subj; subj =~ /__START_SUBJECT__ (.*?) __END_SUBJECT__/ && $1; end
 
-  ## Adds a message to the index. The message cannot already exist in
-  ## the index.
-  def add_message m
-    raise ArgumentError, "index already contains #{m.id}" if contains? m
-
-    source_id = 
-      if m.source.is_a? Integer
-        m.source
-      else
-        m.source.id or raise "unregistered source #{m.source} (id #{m.source.id.inspect})"
-      end
-
-    to = (m.to + m.cc + m.bcc).map { |x| x.email }.join(" ")
-    d = {
-      :message_id => m.id,
-      :source_id => source_id,
-      :source_info => m.source_info,
-      :date => m.date.to_indexable_s,
-      :body => m.content,
-      :snippet => m.snippet,
-      :label => m.labels.join(" "),
-      :from => m.from ? m.from.email : "",
-      :to => (m.to + m.cc + m.bcc).map { |x| x.email }.join(" "),
-      :subject => wrap_subj(Message.normalize_subj(m.subj)),
-      :refs => (m.refs + m.replytos).uniq.join(" "),
-    }
-
-    @index.add_document d
-    
-    docid, entry = load_entry_for_id m.id
-    ## this hasn't been triggered in a long time. TODO: decide whether it's still a problem.
-    raise "just added message #{m.id} but couldn't find it in a search" unless docid
-    true
-  end
-
   def drop_entry docno; @index.delete docno; end
 
   def load_entry_for_id mid
diff --git a/lib/sup/maildir.rb b/lib/sup/maildir.rb
index a76ec60..ba42d79 100644
--- a/lib/sup/maildir.rb
+++ b/lib/sup/maildir.rb
@@ -96,7 +96,7 @@ class Maildir < Source
 private
 
   def die message, opts={}
-    message += " It is likely that messages have been deleted from this Maildir mailbox. Please run sup-import --rebuild #{to_s} to correct this problem." if opts[:suggest_rebuild]
+    message += " It is likely that messages have been deleted from this Maildir mailbox. Please run sup-sync --changed #{to_s} to correct this problem." if opts[:suggest_rebuild]
     self.broken_msg = message
     Redwood::log message
     BufferManager.flash "Error communicating with Maildir. See log for details." if BufferManager.instantiated?
diff --git a/lib/sup/mbox/loader.rb b/lib/sup/mbox/loader.rb
index 538d09d..724d49b 100644
--- a/lib/sup/mbox/loader.rb
+++ b/lib/sup/mbox/loader.rb
@@ -24,7 +24,7 @@ class Loader < Source
     end
 
     if cur_offset > end_offset
-      self.broken_msg = "mbox file is smaller than last recorded message offset. Messages have probably been deleted via another client. Run 'sup-import --rebuild #{to_s}' to correct this."
+      self.broken_msg = "mbox file is smaller than last recorded message offset. Messages have probably been deleted via another client. Run 'sup-sync --changed #{to_s}' to correct this."
     end
   end
 
@@ -38,7 +38,7 @@ class Loader < Source
       l = @f.gets
       unless l =~ BREAK_RE
         Redwood::log "#{to_s}: offset mismatch in mbox file offset #{offset.inspect}: #{l.inspect}"
-        self.broken_msg = "offset mismatch in mbox file offset #{offset.inspect}: #{l.inspect}. Run 'sup-import --rebuild #{to_s}' to correct this." 
+        self.broken_msg = "offset mismatch in mbox file offset #{offset.inspect}: #{l.inspect}. Run 'sup-sync --changed #{to_s}' to correct this." 
         raise SourceError, self.broken_msg
       end
       header = MBox::read_header @f
diff --git a/lib/sup/message.rb b/lib/sup/message.rb
index c392c44..5a9eda7 100644
--- a/lib/sup/message.rb
+++ b/lib/sup/message.rb
@@ -149,7 +149,7 @@ class Message
 
   def save index
     return if broken?
-    index.update_message self if @dirty
+    index.sync_message self if @dirty
     @dirty = false
   end
 
diff --git a/lib/sup/poll.rb b/lib/sup/poll.rb
index e85b019..3483383 100644
--- a/lib/sup/poll.rb
+++ b/lib/sup/poll.rb
@@ -46,7 +46,7 @@ class PollManager
         yield "Loading from #{source}... " unless source.done? || source.broken?
         num = 0
         numi = 0
-        add_new_messages_from source do |m, offset, entry|
+        add_messages_from source do |m, offset, entry|
           ## always preserve the labels on disk.
           m.labels = entry[:label].split(/\s+/).map { |x| x.intern } if entry
           yield "Found message at #{offset} with labels {#{m.labels * ', '}}"
@@ -69,18 +69,19 @@ class PollManager
   end
 
   ## this is the main mechanism for adding new messages to the
-  ## index. it's called both by sup-import and by PollMode.
+  ## index. it's called both by sup-sync and by PollMode.
   ##
-  ## for each new message in the source, this yields the message, the
-  ## source offset, and the index entry on disk (if any). it expects
-  ## the yield to return the message (possibly altered in some way),
-  ## and then adds it (if new) or updates it (if previously seen).
+  ## for each message in the source, starting from the source's
+  ## starting offset, this methods yields the message, the source
+  ## offset, and the index entry on disk (if any). it expects the
+  ## yield to return the message (possibly altered in some way), and
+  ## then adds it (if new) or updates it (if previously seen).
   ##
-  ## the labels of the yielded message are the source labels. it is
-  ## likely that callers will want to replace these with the index
-  ## labels, if they exist, so that state is not lost when e.g. a new
-  ## version of a message from a mailing list comes in.
-  def add_new_messages_from source
+  ## the labels of the yielded message are the default source
+  ## labels. it is likely that callers will want to replace these with
+  ## the index labels, if they exist, so that state is not lost when
+  ## e.g. a new version of a message from a mailing list comes in.
+  def add_messages_from source
     return if source.done? || source.broken?
 
     begin
@@ -101,22 +102,16 @@ class PollManager
           end
 
           docid, entry = Index.load_entry_for_id m.id
-          m = yield m, offset, entry
-          next unless m
-          if entry
-            Index.update_message m, docid, entry
-          else
-            Index.add_message m
-            UpdateManager.relay self, :add, m
-          end
-        rescue MessageFormatError, SourceError => e
+          m = yield(m, offset, entry) or next
+          Index.sync_message m, docid, entry
+          UpdateManager.relay self, :add, m unless entry
+        rescue MessageFormatError => e
           Redwood::log "ignoring erroneous message at #{source}##{offset}: #{e.message}"
-          Redwood::report_broken_sources if BufferManager.instantiated?
         end
       end
     rescue SourceError => e
       Redwood::log "problem getting messages from #{source}: #{e.message}"
-      Redwood::report_broken_sources if BufferManager.instantiated?
+      Redwood::report_broken_sources
     end
   end
 end
diff --git a/lib/sup/sent.rb b/lib/sup/sent.rb
index b28c08d..37b9524 100644
--- a/lib/sup/sent.rb
+++ b/lib/sup/sent.rb
@@ -23,7 +23,7 @@ class SentManager
     end
     @source.each do |offset, labels|
       m = Message.new :source => @source, :source_info => offset, :labels => @source.labels
-      Index.add_message m
+      Index.sync_message m
       UpdateManager.relay self, :add, m
     end
   end
diff --git a/lib/sup/thread.rb b/lib/sup/thread.rb
index 3d94567..85e503c 100644
--- a/lib/sup/thread.rb
+++ b/lib/sup/thread.rb
@@ -54,7 +54,7 @@ class Thread
   ## message can be a Message object, or :fake_root, or nil.
   def each fake_root=false
     adj = 0
-    root = @containers.find_all { |c| !Message.subj_is_reply?(c) }.argmin { |c| c.date }
+    root = @containers.find_all { |c| !Message.subj_is_reply?(c) }.argmin { |c| c.date || 0 }
 
     if root
       adj = 1