From 07b1fdc67e634001857aa5a252f5a3a31124bb3b Mon Sep 17 00:00:00 2001 From: wmorgan Date: Tue, 6 Feb 2007 19:26:45 +0000 Subject: [PATCH] fixed polling (yet again) git-svn-id: svn://rubyforge.org/var/svn/sup/trunk@305 5c8cc53c-5e98-4d25-b20a-d8db53a31250 --- bin/sup-import | 17 ++++++++++----- lib/sup/index.rb | 47 +++------------------------------------ lib/sup/poll.rb | 57 +++++++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 71 insertions(+), 50 deletions(-) diff --git a/bin/sup-import b/bin/sup-import index f801246..69b5b07 100644 --- a/bin/sup-import +++ b/bin/sup-import @@ -81,13 +81,18 @@ found = {} begin sources.each do |source| num = 0 - index.add_new_messages_from source do |m, offset, source_labels, entry| + Redwood::PollManager.add_new_messages_from source do |m, offset, entry| found[m.id] = true - m.labels = source_labels if opts[:overwrite_state] - m.labels -= [:inbox] if opts[:archive] - m.labels -= [:unread] if opts[:read] - num += 1 + ## if the entry exists on disk + if entry && !opts[:overwrite_state] + m.labels = entry[:label].split(/\s+/).map { |x| x.intern } + else + ## m.labels defaults to labels from the source + m.labels -= [:inbox] if opts[:archive] + m.labels -= [:unread] if opts[:read] + end + if num % 1000 == 0 && num > 0 elapsed = Time.now - start pctdone = source.pct_done @@ -98,10 +103,12 @@ begin ## update if... if entry.nil? # it's a new message; or puts "# adding message at #{offset}, labels: #{m.labels * ' '}" if opts[:verbose] + num += 1 m elsif opts[:full_rebuild] || # we're updating everyone; or (opts[:rebuild] && (entry[:source_id].to_i != source.id || entry[:source_info].to_i != offset)) # we're updating just the changed ones puts "# updating message at #{offset}, source #{entry[:source_id]} => #{source.id}, offset #{entry[:source_info]} => #{offset}, labels: #{m.labels * ' '}" if opts[:verbose] + num += 1 m else nil diff --git a/lib/sup/index.rb b/lib/sup/index.rb index 16fd3b9..28fde2a 100644 --- a/lib/sup/index.rb +++ b/lib/sup/index.rb @@ -75,6 +75,9 @@ class Index ## Update the message state on disk, by deleting and re-adding it. ## The message must exist in the index. docid and entry are found ## unless given. + ## + ## Overwrites the labels on disk with the new labels in 'm', so that + ## we can actually change message state. def update_message m, docid=nil, entry=nil unless docid && entry docid, entry = load_entry_for_id m.id @@ -85,55 +88,11 @@ class Index raise "deleting non-corresponding entry #{docid}" unless @index[docid][:message_id] == m.id - m.labels = entry[:label].split(/\s+/).map { |x| x.intern } @index.delete docid add_message m docid, entry = load_entry_for_id m.id end - ## for each new message form the source, yields a bunch of stuff, - ## gets the message back from the block, and adds it or updates it. - def add_new_messages_from source - found = {} - return if source.done? || source.broken? - - source.each do |offset, labels| - if source.broken? - Redwood::log "error loading messages from #{source}: #{source.broken_msg}" - return - end - - labels.each { |l| LabelManager << l } - - begin - m = Message.new :source => source, :source_info => offset, :labels => labels - if found[m.id] - Redwood::log "skipping duplicate message #{m.id}" - next - else - found[m.id] = true - end - - if m.source_marked_read? - m.remove_label :unread - labels.delete :unread - end - - docid, entry = load_entry_for_id m.id - m = yield m, offset, labels, entry - next unless m - if entry - update_message m, docid, entry - else - add_message m - UpdateManager.relay :add, m - end - rescue MessageFormatError, SourceError => e - Redwood::log "ignoring erroneous message at #{source}##{offset}: #{e.message}" - end - end - end - def save_index fn=File.join(@dir, "ferret") # don't have to do anything, apparently end diff --git a/lib/sup/poll.rb b/lib/sup/poll.rb index 9fc8b6f..3157972 100644 --- a/lib/sup/poll.rb +++ b/lib/sup/poll.rb @@ -46,7 +46,9 @@ class PollManager yield "Loading from #{source}... " unless source.done? || source.broken? num = 0 numi = 0 - Index.add_new_messages_from source do |m, offset, source_labels, entry| + add_new_messages_from source do |m, offset, entry| + ## always preserve the labels on disk. + m.labels = entry[:label].split(/\s+/).map { |x| x.intern } if entry yield "Found message at #{offset} with labels #{m.labels * ', '}" num += 1 numi += 1 if m.labels.include? :inbox @@ -63,6 +65,59 @@ class PollManager end [total_num, total_numi] end + + ## this is the main mechanism for adding new messages to the + ## index. it's called both by sup-import and by PollMode. + ## + ## for each new message in the source, this yields the message, the + ## source offset, and the index entry on disk (if any). it expects + ## the yield to return the message (possibly altered in some way), + ## and then adds it (if new) or updates it (if previously seen). + ## + ## the labels of the yielded message are the source labels. it is + ## likely that callers will want to replace these with the index + ## labels, if they exist, so that state is not lost when e.g. a new + ## version of a message from a mailing list comes in. + def add_new_messages_from source + found = {} + return if source.done? || source.broken? + + source.each do |offset, labels| + if source.broken? + Redwood::log "error loading messages from #{source}: #{source.broken_msg}" + return + end + + labels.each { |l| LabelManager << l } + + begin + m = Message.new :source => source, :source_info => offset, :labels => labels + if found[m.id] + Redwood::log "skipping duplicate message #{m.id}" + next + else + found[m.id] = true + end + + if m.source_marked_read? + m.remove_label :unread + labels.delete :unread + end + + docid, entry = Index.load_entry_for_id m.id + m = yield m, offset, entry + next unless m + if entry + update_message m, docid, entry + else + add_message m + UpdateManager.relay :add, m + end + rescue MessageFormatError, SourceError => e + Redwood::log "ignoring erroneous message at #{source}##{offset}: #{e.message}" + end + end + end end end -- 2.43.0