8 PROGRESS_UPDATE_INTERVAL = 15 # seconds
11 def to_s; sprintf '%.2f', self; end
12 def to_time_s; infinite? ? "unknown" : super end
18 sprintf "%d:%02d:%02d", i / 3600, (i / 60) % 60, i % 60
23 def to_s; to_a * ',' end
32 opts = Trollop::options do
33 version "sup-sync (sup #{Redwood::VERSION})"
35 Synchronizes the Sup index with one or more message sources by adding
36 messages, deleting messages, or changing message state in the index as
39 "Message state" means read/unread, archived/inbox, starred/unstarred,
40 and all user-defined labels on each message.
42 "Default source state" refers to any state that a source itself has
43 keeps about a message. Sup-sync uses this information when adding a
44 new message to the index. The source state is typically limited to
45 read/unread, archived/inbox status and a single label based on the
46 source name. Messages using the default source state are placed in
47 the inbox (i.e. not archived) and unstarred.
50 sup-sync [options] <source>*
52 where <source>* is zero or more source URIs. If no sources are given,
53 sync from all usual sources. Supported source URI schemes can be seen
54 by running "sup-add --help".
56 Options controlling WHICH messages sup-sync operates on:
58 opt :new, "Operate on new messages only. Don't scan over the entire source. (Default.)", :short => :none
59 opt :changed, "Scan over the entire source for messages that have been deleted, altered, or moved from another source."
60 opt :restored, "Operate only on those messages included in a dump file as specified by --restore which have changed state."
61 opt :all, "Operate on all messages in the source, regardless of newness or changedness."
62 opt :start_at, "For --changed, --restored and --all, start at a particular offset.", :type => :int
66 Options controlling HOW message state is altered:
68 opt :asis, "If the message is already in the index, preserve its state. Otherwise, use default source state. (Default.)", :short => :none
69 opt :restore, "Restore message state from a dump file created with sup-dump. If a message is not in this dumpfile, act as --asis.", :type => String, :short => :none
70 opt :discard, "Discard any message state in the index and use the default source state. Dangerous!", :short => :none
71 opt :archive, "When using the default source state, mark messages as archived.", :short => "-x"
72 opt :read, "When using the default source state, mark messages as read."
73 opt :extra_labels, "When using the default source state, also apply these user-defined labels (a comma-separated list)", :default => "", :short => :none
79 opt :verbose, "Print message ids as they're processed."
80 opt :optimize, "As the final operation, optimize the index."
81 opt :all_sources, "Scan over all sources.", :short => :none
82 opt :dry_run, "Don't actually modify the index. Probably only useful with --verbose.", :short => "-n"
83 opt :version, "Show version information", :short => :none
85 conflicts :changed, :all, :new, :restored
86 conflicts :asis, :restore, :discard
88 Trollop::die :restored, "requires --restore" if opts[:restored] unless opts[:restore]
90 Trollop::die :start_at, "must be non-negative" if opts[:start_at] < 0
91 Trollop::die :start_at, "requires either --changed, --restored or --all" unless opts[:changed] || opts[:restored] || opts[:all]
94 target = [:new, :changed, :all, :restored].find { |x| opts[x] } || :new
95 op = [:asis, :restore, :discard].find { |x| opts[x] } || :asis
98 index = Redwood::Index.init
100 restored_state = if opts[:restore]
102 $stderr.puts "Loading state dump from #{opts[:restore]}..."
103 IO.foreach opts[:restore] do |l|
104 l =~ /^(\S+) \((.*?)\)$/ or raise "Can't read dump line: #{l.inspect}"
106 dump[mid] = labels.to_set_of_symbols
108 $stderr.puts "Read #{dump.size} entries from dump file."
119 sources = if opts[:all_sources]
120 Redwood::SourceManager.sources
122 Redwood::SourceManager.usual_sources
125 Redwood::SourceManager.source_for uri or Trollop::die "Unknown source: #{uri}. Did you add it with sup-add first?"
129 ## for all target specifications except for only-new messages, reset the
130 ## source to the beginning (or to the user-specified starting point.)
131 unless target == :new
133 Trollop::die :start_at, "can only be used on one source" unless sources.size == 1
134 sources.first.seek_to! opts[:start_at]
135 sources.first.correct_offset! if sources.first.respond_to?(:correct_offset!)
137 sources.each { |s| s.reset! }
141 sources.each do |source|
142 $stderr.puts "Scanning #{source}..."
143 num_added = num_updated = num_scanned = num_restored = 0
144 last_info_time = start_time = Time.now
146 Redwood::PollManager.each_message_from source do |m|
149 old_m = index.build_message m.id
153 ## skip this message if we're operating only on changed messages, the
154 ## message is in the index, and it's unchanged from what the source is
156 next if old_m && old_m.source.id == m.source.id && old_m.source_info == m.source_info
158 ## skip if we're operating on restored messages, and this one
159 ## ain't (or we wouldn't be making a change)
160 next unless old_m && restored_state[m.id] && restored_state[m.id] != old_m.labels
162 ## nothing to do; we'll consider all messages starting at the start offset, which
163 ## hasn't been changed.
165 ## nothing to do; we'll consider all messages starting at the start offset, which
166 ## was reset to the beginning above.
169 ## tweak source labels according to commandline arguments if necessary
170 m.labels.delete :inbox if opts[:archive]
171 m.labels.delete :unread if opts[:read]
172 m.labels += opts[:extra_labels].to_set_of_symbols(",")
174 ## decide what to do based on message labels and the operation we're performing
175 dothis, new_labels = case
176 when (op == :restore) && restored_state[m.id] && old_m && (old_m.labels != restored_state[m.id])
177 [:update_message_state, restored_state[m.id]]
179 if old_m && (old_m.labels != m.labels)
180 [:update_message_state, m.labels]
185 ## duplicate behavior of poll mode: if index_state is non-nil, this is a newer
186 ## version of an older message, so merge in any new labels except :unread and
189 ## TODO: refactor such that this isn't duplicated
191 m.labels = old_m.labels + (m.labels - [:unread, :inbox])
198 ## now, actually do the operation
201 $stderr.puts "Adding new message #{source}###{m.source_info} with labels #{m.labels}" if opts[:verbose]
202 index.add_message m unless opts[:dry_run]
205 $stderr.puts "Updating message #{source}###{m.source_info}; labels #{old_m.labels} => #{m.labels}; offset #{old_m.source_info} => #{m.source_info}" if opts[:verbose]
206 index.update_message m unless opts[:dry_run]
208 when :update_message_state
209 $stderr.puts "Changing flags for #{source}##{m.source_info} from #{m.labels} to #{new_labels}"
210 m.labels = new_labels
211 index.update_message_state m unless opts[:dry_run]
215 if Time.now - last_info_time > PROGRESS_UPDATE_INTERVAL
216 last_info_time = Time.now
217 elapsed = last_info_time - start_time
218 pctdone = source.respond_to?(:pct_done) ? source.pct_done : 100.0 * (source.cur_offset.to_f - source.start_offset).to_f / (source.end_offset - source.start_offset).to_f
219 remaining = (100.0 - pctdone) * (elapsed.to_f / pctdone)
220 $stderr.printf "## read %dm (about %.0f%%) @ %.1fm/s. %s elapsed, about %s remaining\n", num_scanned, pctdone, num_scanned / elapsed, elapsed.to_time_s, remaining.to_time_s
224 $stderr.puts "Scanned #{num_scanned}, added #{num_added}, updated #{num_updated} messages from #{source}."
225 $stderr.puts "Restored state on #{num_restored} (#{100.0 * num_restored / num_scanned}%) messages." if num_restored > 0
228 ## delete any messages in the index that claim they're from one of
229 ## these sources, but that we didn't see.
230 if (target == :all || target == :changed)
231 $stderr.puts "Deleting missing messages from the index..."
232 num_del, num_scanned = 0, 0
233 sources.each do |source|
234 raise "no source id for #{source}" unless source.id
235 index.each_message :source_id => source.id, :load_spam => true, :load_deleted => true, :load_killed => true do |m|
238 next unless m.source_info >= opts[:start_at] if opts[:start_at]
239 puts "Deleting #{m.id}" if opts[:verbose]
240 index.delete m.id unless opts[:dry_run]
245 $stderr.puts "Deleted #{num_del} / #{num_scanned} messages"
251 $stderr.puts "Optimizing index..."
252 optt = time { index.optimize unless opts[:dry_run] }
253 $stderr.puts "Optimized index of size #{index.size} in #{optt}s."
255 rescue Redwood::FatalSourceError => e
256 $stderr.puts "Sorry, I couldn't communicate with a source: #{e.message}"
257 rescue Exception => e
258 File.open("sup-exception-log.txt", "w") { |f| f.puts e.backtrace }