8 PROGRESS_UPDATE_INTERVAL = 15 # seconds
11 def to_s; sprintf '%.2f', self; end
13 infinite? ? "unknown" : super
20 sprintf "%d:%02d:%02d", i / 3600, (i / 60) % 60, i % 60
30 opts = Trollop::options do
31 version "sup-sync (sup #{Redwood::VERSION})"
33 Synchronizes the Sup index with one or more message sources by adding
34 messages, deleting messages, or changing message state in the index as
37 "Message state" means read/unread, archived/inbox, starred/unstarred,
38 and all user-defined labels on each message.
40 "Default source state" refers to any state that a source itself has
41 keeps about a message. Sup-sync uses this information when adding a
42 new message to the index. The source state is typically limited to
43 read/unread, archived/inbox status and a single label based on the
44 source name. Messages using the default source state are placed in
45 the inbox (i.e. not archived) and unstarred.
48 sup-sync [options] <source>*
50 where <source>* is zero or more source URIs. If no sources are given,
51 sync from all usual sources. Supported source URI schemes can be seen
52 by running "sup-add --help".
54 Options controlling WHICH messages sup-sync operates on:
56 opt :new, "Operate on new messages only. Don't scan over the entire source. (Default.)", :short => :none
57 opt :changed, "Scan over the entire source for messages that have been deleted, altered, or moved from another source. (In the case of mbox sources, this includes all messages AFTER an altered message.)"
58 opt :restored, "Operate only on those messages included in a dump file as specified by --restore which have changed state."
59 opt :all, "Operate on all messages in the source, regardless of newness or changedness."
60 opt :start_at, "For --changed, --restored and --all, start at a particular offset.", :type => :int
64 Options controlling HOW message state is altered:
66 opt :asis, "If the message is already in the index, preserve its state. Otherwise, use default source state. (Default.)", :short => :none
67 opt :restore, "Restore message state from a dump file created with sup-dump. If a message is not in this dumpfile, act as --asis.", :type => String, :short => :none
68 opt :discard, "Discard any message state in the index and use the default source state. Dangerous!", :short => :none
69 opt :archive, "When using the default source state, mark messages as archived.", :short => "-x"
70 opt :read, "When using the default source state, mark messages as read."
71 opt :extra_labels, "When using the default source state, also apply these user-defined labels. Should be a comma-separated list.", :type => String, :short => :none
77 opt :verbose, "Print message ids as they're processed."
78 opt :optimize, "As the final operation, optimize the index."
79 opt :all_sources, "Scan over all sources.", :short => :none
80 opt :dry_run, "Don't actually modify the index. Probably only useful with --verbose.", :short => "-n"
81 opt :version, "Show version information", :short => :none
83 conflicts :changed, :all, :new, :restored
84 conflicts :asis, :restore, :discard
86 Trollop::die :restored, "requires --restore" if opts[:restored] unless opts[:restore]
88 Trollop::die :start_at, "must be non-negative" if opts[:start_at] < 0
89 Trollop::die :start_at, "requires either --changed, --restored or --all" unless opts[:changed] || opts[:restored] || opts[:all]
92 target = [:new, :changed, :all, :restored].find { |x| opts[x] } || :new
93 op = [:asis, :restore, :discard].find { |x| opts[x] } || :asis
96 index = Redwood::Index.new
101 $stderr.puts "Loading state dump from #{opts[:restore]}..."
102 IO.foreach opts[:restore] do |l|
103 l =~ /^(\S+) \((.*?)\)$/ or raise "Can't read dump line: #{l.inspect}"
105 dump[mid] = labels.to_set_of_symbols
107 $stderr.puts "Read #{dump.size} entries from dump file."
118 sources = ARGV.map do |uri|
119 Redwood::SourceManager.source_for uri or Trollop::die "Unknown source: #{uri}. Did you add it with sup-add first?"
122 sources = Redwood::SourceManager.usual_sources if sources.empty?
123 sources = Redwood::SourceManager.sources if opts[:all_sources]
125 unless target == :new
127 Trollop::die :start_at, "can only be used on one source" unless sources.size == 1
128 sources.first.seek_to! opts[:start_at]
129 sources.first.correct_offset! if sources.first.respond_to?(:correct_offset!)
131 sources.each { |s| s.reset! }
135 sources.each do |source|
136 $stderr.puts "Scanning #{source}..."
137 num_added = num_updated = num_scanned = num_restored = 0
138 last_info_time = start_time = Time.now
140 Redwood::PollManager.add_messages_from source, :force_overwrite => true do |m_old, m, offset|
144 if Time.now - last_info_time > PROGRESS_UPDATE_INTERVAL
145 last_info_time = Time.now
146 elapsed = last_info_time - start_time
147 start = opts[:start_at] || source.start_offset
148 pctdone = 100.0 * (source.cur_offset - start).to_f / (source.end_offset - start).to_f
149 remaining = (100.0 - pctdone) * (elapsed.to_f / pctdone)
150 $stderr.printf "## read %dm (about %.0f%%) @ %.1fm/s. %s elapsed, about %s remaining\n", num_scanned, pctdone, num_scanned / elapsed, elapsed.to_time_s, remaining.to_time_s
153 ## skip if we're operating only on changed messages, the message
154 ## is in the index, and it's unchanged from what the source is
156 next if target == :changed && m_old && m_old.source.id == source.id && m_old.source_info == offset
158 ## get the state currently in the index
159 index_state = m_old.labels.dup if m_old
161 ## skip if we're operating on restored messages, and this one
162 ## ain't (or we wouldn't be making a change)
163 next if target == :restored && (!restored_state[m.id] || !index_state || restored_state[m.id] == index_state)
165 ## m.labels is the default source labels. tweak these according
166 ## to default source state modification flags.
167 m.labels.delete :inbox if opts[:archive]
168 m.labels.delete :unread if opts[:read]
169 m.labels += opts[:extra_labels].to_set_of_symbols(",") if opts[:extra_labels]
171 ## assign message labels based on the operation we're performing
174 ## duplicate behavior of poll mode: if index_state is non-nil, this is a newer
175 ## version of an older message, so merge in any new labels except :unread and
177 m.labels = ((m.labels - [:unread, :inbox]) + index_state) if index_state
179 ## if the entry exists on disk
180 if restored_state[m.id]
181 m.labels = restored_state[m.id]
184 m.labels = index_state
187 ## nothin! use default source labels
190 if Time.now - last_info_time > PROGRESS_UPDATE_INTERVAL
191 last_info_time = Time.now
192 elapsed = last_info_time - start_time
193 pctdone = source.respond_to?(:pct_done) ? source.pct_done : 100.0 * (source.cur_offset.to_f - source.start_offset).to_f / (source.end_offset - source.start_offset).to_f
194 remaining = (100.0 - pctdone) * (elapsed.to_f / pctdone)
195 $stderr.printf "## read %dm (about %.0f%%) @ %.1fm/s. %s elapsed, about %s remaining\n", num_scanned, pctdone, num_scanned / elapsed, elapsed.to_time_s, remaining.to_time_s
199 puts "Adding message #{source}##{offset} from #{m.from} with state {#{m.labels.to_a * ', '}}" if opts[:verbose]
202 puts "Updating message #{source}##{offset}, source #{m_old.source.id} => #{source.id}, offset #{m_old.source_info} => #{offset}, state {#{index_state.to_a * ', '}} => {#{m.labels.to_a * ', '}}" if opts[:verbose]
206 opts[:dry_run] ? nil : m
208 $stderr.puts "Scanned #{num_scanned}, added #{num_added}, updated #{num_updated} messages from #{source}."
209 $stderr.puts "Restored state on #{num_restored} (#{100.0 * num_restored / num_scanned}%) messages." if num_restored > 0
212 ## delete any messages in the index that claim they're from one of
213 ## these sources, but that we didn't see.
214 if (target == :all || target == :changed)
215 $stderr.puts "Deleting missing messages from the index..."
216 num_del, num_scanned = 0, 0
217 sources.each do |source|
218 raise "no source id for #{source}" unless source.id
219 index.each_message :source_id => source.id, :load_spam => true, :load_deleted => true, :load_killed => true do |m|
222 next unless m.source_info >= opts[:start_at] if opts[:start_at]
223 puts "Deleting #{m.id}" if opts[:verbose]
224 index.delete m.id unless opts[:dry_run]
229 $stderr.puts "Deleted #{num_del} / #{num_scanned} messages"
235 $stderr.puts "Optimizing index..."
236 optt = time { index.optimize unless opts[:dry_run] }
237 $stderr.puts "Optimized index of size #{index.size} in #{optt}s."
239 rescue Redwood::FatalSourceError => e
240 $stderr.puts "Sorry, I couldn't communicate with a source: #{e.message}"
241 rescue Exception => e
242 File.open("sup-exception-log.txt", "w") { |f| f.puts e.backtrace }