begin
sources.each do |source|
num = 0
- index.add_new_messages_from source do |m, offset, source_labels, entry|
+ Redwood::PollManager.add_new_messages_from source do |m, offset, entry|
found[m.id] = true
- m.labels = source_labels if opts[:overwrite_state]
- m.labels -= [:inbox] if opts[:archive]
- m.labels -= [:unread] if opts[:read]
- num += 1
+ ## if the entry exists on disk
+ if entry && !opts[:overwrite_state]
+ m.labels = entry[:label].split(/\s+/).map { |x| x.intern }
+ else
+ ## m.labels defaults to labels from the source
+ m.labels -= [:inbox] if opts[:archive]
+ m.labels -= [:unread] if opts[:read]
+ end
+
if num % 1000 == 0 && num > 0
elapsed = Time.now - start
pctdone = source.pct_done
## update if...
if entry.nil? # it's a new message; or
puts "# adding message at #{offset}, labels: #{m.labels * ' '}" if opts[:verbose]
+ num += 1
m
elsif opts[:full_rebuild] || # we're updating everyone; or
(opts[:rebuild] && (entry[:source_id].to_i != source.id || entry[:source_info].to_i != offset)) # we're updating just the changed ones
puts "# updating message at #{offset}, source #{entry[:source_id]} => #{source.id}, offset #{entry[:source_info]} => #{offset}, labels: #{m.labels * ' '}" if opts[:verbose]
+ num += 1
m
else
nil
## Update the message state on disk, by deleting and re-adding it.
## The message must exist in the index. docid and entry are found
## unless given.
+ ##
+ ## Overwrites the labels on disk with the new labels in 'm', so that
+ ## we can actually change message state.
def update_message m, docid=nil, entry=nil
unless docid && entry
docid, entry = load_entry_for_id m.id
raise "deleting non-corresponding entry #{docid}" unless @index[docid][:message_id] == m.id
- m.labels = entry[:label].split(/\s+/).map { |x| x.intern }
@index.delete docid
add_message m
docid, entry = load_entry_for_id m.id
end
- ## for each new message form the source, yields a bunch of stuff,
- ## gets the message back from the block, and adds it or updates it.
- def add_new_messages_from source
- found = {}
- return if source.done? || source.broken?
-
- source.each do |offset, labels|
- if source.broken?
- Redwood::log "error loading messages from #{source}: #{source.broken_msg}"
- return
- end
-
- labels.each { |l| LabelManager << l }
-
- begin
- m = Message.new :source => source, :source_info => offset, :labels => labels
- if found[m.id]
- Redwood::log "skipping duplicate message #{m.id}"
- next
- else
- found[m.id] = true
- end
-
- if m.source_marked_read?
- m.remove_label :unread
- labels.delete :unread
- end
-
- docid, entry = load_entry_for_id m.id
- m = yield m, offset, labels, entry
- next unless m
- if entry
- update_message m, docid, entry
- else
- add_message m
- UpdateManager.relay :add, m
- end
- rescue MessageFormatError, SourceError => e
- Redwood::log "ignoring erroneous message at #{source}##{offset}: #{e.message}"
- end
- end
- end
-
def save_index fn=File.join(@dir, "ferret")
# don't have to do anything, apparently
end
yield "Loading from #{source}... " unless source.done? || source.broken?
num = 0
numi = 0
- Index.add_new_messages_from source do |m, offset, source_labels, entry|
+ add_new_messages_from source do |m, offset, entry|
+ ## always preserve the labels on disk.
+ m.labels = entry[:label].split(/\s+/).map { |x| x.intern } if entry
yield "Found message at #{offset} with labels #{m.labels * ', '}"
num += 1
numi += 1 if m.labels.include? :inbox
end
[total_num, total_numi]
end
+
+ ## this is the main mechanism for adding new messages to the
+ ## index. it's called both by sup-import and by PollMode.
+ ##
+ ## for each new message in the source, this yields the message, the
+ ## source offset, and the index entry on disk (if any). it expects
+ ## the yield to return the message (possibly altered in some way),
+ ## and then adds it (if new) or updates it (if previously seen).
+ ##
+ ## the labels of the yielded message are the source labels. it is
+ ## likely that callers will want to replace these with the index
+ ## labels, if they exist, so that state is not lost when e.g. a new
+ ## version of a message from a mailing list comes in.
+ def add_new_messages_from source
+ found = {}
+ return if source.done? || source.broken?
+
+ source.each do |offset, labels|
+ if source.broken?
+ Redwood::log "error loading messages from #{source}: #{source.broken_msg}"
+ return
+ end
+
+ labels.each { |l| LabelManager << l }
+
+ begin
+ m = Message.new :source => source, :source_info => offset, :labels => labels
+ if found[m.id]
+ Redwood::log "skipping duplicate message #{m.id}"
+ next
+ else
+ found[m.id] = true
+ end
+
+ if m.source_marked_read?
+ m.remove_label :unread
+ labels.delete :unread
+ end
+
+ docid, entry = Index.load_entry_for_id m.id
+ m = yield m, offset, entry
+ next unless m
+ if entry
+ update_message m, docid, entry
+ else
+ add_message m
+ UpdateManager.relay :add, m
+ end
+ rescue MessageFormatError, SourceError => e
+ Redwood::log "ignoring erroneous message at #{source}##{offset}: #{e.message}"
+ end
+ end
+ end
end
end
projects / sup / commitdiff