From: wmorgan Date: Sun, 10 Dec 2006 19:13:21 +0000 (+0000) Subject: (large change) enabled imap support, and reworked everything to allow for X-Git-Url: https://git.cworth.org/git?a=commitdiff_plain;h=4373f679320b906d4af071ccc5cbed56a865c519;p=sup (large change) enabled imap support, and reworked everything to allow for loading of message from sources that are broken. so if the imap server is dead, you still have a somewhat reaosnalbe interface, based on everything it knows about the messages from the index. git-svn-id: svn://rubyforge.org/var/svn/sup/trunk@70 5c8cc53c-5e98-4d25-b20a-d8db53a31250 --- diff --git a/bin/sup-import b/bin/sup-import index c7fe2ad..2306694 100644 --- a/bin/sup-import +++ b/bin/sup-import @@ -85,6 +85,7 @@ index.load puts "loaded index of #{index.size} messages" sources = ARGV.map do |fn| + fn = "mbox://#{fn}" unless fn =~ %r!://! source = index.source_for fn unless source source = @@ -133,7 +134,7 @@ begin found[m.id] = true end m.remove_label :unread if m.status == "RO" unless force_read - puts "# message at #{offset} labels #{labels.inspect}" unless rebuild || force_rebuild + puts "# message at #{offset}, labels: #{labels * ', '}" unless rebuild || force_rebuild if (rebuild || force_rebuild) && (docid, entry = index.load_entry_for_id(m.id)) && entry if force_rebuild || entry[:source_info].to_i != offset @@ -144,7 +145,7 @@ begin else num += 1 if index.add_message m end - rescue Redwood::MessageFormatError, Redwood::MBox::Error => e + rescue Redwood::MessageFormatError, Redwood::SourceError => e $stderr.puts "ignoring erroneous message at #{source}##{offset}: #{e.message}" end if num % 1000 == 0 && num > 0 diff --git a/lib/sup.rb b/lib/sup.rb index 703b396..5bf3928 100644 --- a/lib/sup.rb +++ b/lib/sup.rb @@ -3,12 +3,13 @@ require 'yaml' require 'zlib' require 'thread' require 'fileutils' + Thread.abort_on_exception = true # make debugging possible class Object - ## this is for debugging purposes because i keep calling nil.id and - ## i want it to throw an exception - def id + ## this is for debugging purposes because i keep calling #id on the + ## wrong object and i want it to throw an exception + def id raise "wrong id called" end end @@ -90,6 +91,7 @@ end require "sup/util" require "sup/update" require "sup/message" +require "sup/source" require "sup/mbox" require "sup/imap" require "sup/person" diff --git a/lib/sup/draft.rb b/lib/sup/draft.rb index 5f8cce7..d53963a 100644 --- a/lib/sup/draft.rb +++ b/lib/sup/draft.rb @@ -10,9 +10,9 @@ class DraftManager self.class.i_am_the_instance self end - def self.source_name; "drafts"; end + def self.source_name; "drafts://"; end def self.source_id; 9999; end - def new_source; @source = DraftLoader.new @dir; end + def new_source; @source = DraftLoader.new; end def write_draft offset = @source.gen_offset @@ -36,25 +36,30 @@ class DraftManager end end -class DraftLoader - attr_accessor :dir, :end_offset - bool_reader :dirty +class DraftLoader < Source + attr_accessor :dir - def initialize dir, end_offset=0 + def initialize cur_offset=0 + dir = Redwood::DRAFT_DIR Dir.mkdir dir unless File.exists? dir + super "draft://#{dir}", cur_offset, true, false @dir = dir - @end_offset = end_offset - @dirty = false end - def done?; !File.exists? fn_for_offset(@end_offset); end - def usual?; true; end def id; DraftManager.source_id; end def to_s; DraftManager.source_name; end - def is_source_for? x; x == DraftManager.source_name; end + + def next + ret = nil + begin + ret = cur_offset + self.cur_offset = cur_offset + 1 + end until File.exists? fn_for_offset(ret) + [ret, [:draft]] + end def gen_offset - i = @end_offset + i = cur_offset while File.exists? fn_for_offset(i) i += 1 end @@ -95,18 +100,10 @@ class DraftLoader ret end - def each - while File.exists?(fn = File.join(@dir, @end_offset.to_s)) - yield @end_offset, [:draft, :inbox] - @end_offset += 1 - @dirty = true - end - end - - def total; Dir[File.join(@dir, "*")].sort.last.to_i; end - def reset!; @end_offset = 0; @dirty = true; end + def start_offset; 0; end + def end_offset; Dir.new(@dir).entries.sort.last.to_i + 1; end end -Redwood::register_yaml(DraftLoader, %w(dir end_offset)) +Redwood::register_yaml(DraftLoader, %w(cur_offset)) end diff --git a/lib/sup/imap.rb b/lib/sup/imap.rb index a4559c3..c2e9e6d 100644 --- a/lib/sup/imap.rb +++ b/lib/sup/imap.rb @@ -4,50 +4,63 @@ require 'stringio' module Redwood -class IMAP - attr_reader :uri - bool_reader :usual, :archived, :read, :dirty - attr_accessor :id, :labels - - class Error < StandardError; end - +class IMAP < Source + attr_reader :labels, :broken_msg + def initialize uri, username, password, last_uid=nil, usual=true, archived=false, id=nil - raise "username and password must be specified" unless username && password + raise ArgumentError, "username and password must be specified" unless username && password + + super uri, last_uid, usual, archived, id - @uri_s = uri - @uri = URI(uri) + @parsed_uri = URI(uri) @username = username @password = password - @last_uid = last_uid || 1 - @dirty = false - @usual = usual - @archived = archived - @id = id @imap = nil - @labels = [:unread, - archived ? nil : :inbox, - mailbox !~ /inbox/i && !mailbox.empty? ? mailbox.intern : nil, - ].compact + @labels = [] + @labels << mailbox.intern unless mailbox =~ /inbox/i || mailbox.empty? + @labels << :inbox unless archived? + + connect end def connect - return if @imap - Redwood::log "connecting to #{@uri.host} port #{ssl? ? 993 : 143}, ssl=#{ssl?}" - #raise "simulated imap failure" - @imap = Net::IMAP.new @uri.host, ssl? ? 993 : 143, ssl? - @imap.authenticate('LOGIN', @username, @password) - Redwood::log "success. selecting #{mailbox.inspect}." - @imap.examine(mailbox) + return false if broken? + return true if @imap + Redwood::log "connecting to #{@parsed_uri.host} port #{ssl? ? 993 : 143}, ssl=#{ssl?} ..." + + ## ok, this is FUCKING ANNOYING. + ## + ## what imap.rb likes to do is, if an exception occurs, catch it + ## and re-raise it on the calling thread. seems reasonable. but + ## what that REALLY means is that the only way to reasonably + ## initialize imap is in its own thread, because otherwise, you + ## will never be able to catch the exception it raises on the + ## calling thread, and the backtrace will not make any sense at + ## all, and you will waste HOURS of your life on this fucking + ## problem. + ## + ## FUCK!!!!!!!!! + ::Thread.new do + begin + #raise Net::IMAP::ByeResponseError, "simulated imap failure" + @imap = Net::IMAP.new @parsed_uri.host, ssl? ? 993 : 143, ssl? + @imap.authenticate 'LOGIN', @username, @password + @imap.examine mailbox + Redwood::log "successfully connected to #{@parsed_uri}, mailbox #{mailbox}" + rescue Exception => e + self.broken = true + @imap = nil + @broken_msg = e.message.chomp # fucking chomp! fuck!!! + Redwood::log "error connecting to IMAP server: #{@broken_msg}" + end + end.join + + !!@imap end private :connect - def mailbox; @uri.path[1..-1] end ##XXXX TODO handle nil - def ssl?; @uri.scheme == 'imaps' end - def reset!; @last_uid = 1; @dirty = true; end - def == o; o.is_a?(IMAP) && o.uri == uri; end - def uri; @uri.to_s; end - def to_s; uri; end - def is_source_for? s; to_s == s; end + def mailbox; @parsed_uri.path[1..-1] end ##XXXX TODO handle nil + def ssl?; @parsed_uri.scheme == 'imaps' end def load_header uid=nil MBox::read_header StringIO.new(raw_header(uid)) @@ -59,33 +72,36 @@ class IMAP ## load the full header text def raw_header uid - connect + begin + connect or return broken_msg + rescue Exception => e + raise "wtf: #{e.inspect}" + end @imap.uid_fetch(uid, 'RFC822.HEADER')[0].attr['RFC822.HEADER'].gsub(/\r\n/, "\n") end def raw_full_message uid - connect + connect or return broken_msg @imap.uid_fetch(uid, 'RFC822')[0].attr['RFC822'].gsub(/\r\n/, "\n") end def each - connect - uids = @imap.uid_search ['UID', "#{@last_uid}:#{total}"] + connect or return broken_msg + uids = @imap.uid_search ['UID', "#{cur_offset}:#{end_offset}"] uids.each do |uid| - yield uid, labels @last_uid = uid @dirty = true + yield uid, labels end end - def done?; @last_uid >= total; end - - def total - connect + def start_offset; 1; end + def end_offset + connect or return start_offset @imap.uid_search(['ALL']).last end end -Redwood::register_yaml(IMAP, %w(uri_s username password last_uid usual archived id)) +Redwood::register_yaml(IMAP, %w(uri username password offset usual archived id)) end diff --git a/lib/sup/index.rb b/lib/sup/index.rb index 6b41dd1..4288549 100644 --- a/lib/sup/index.rb +++ b/lib/sup/index.rb @@ -185,25 +185,50 @@ class Index source = @sources[doc[:source_id].to_i] #puts "building message #{doc[:message_id]} (#{source}##{doc[:source_info]})" raise "invalid source #{doc[:source_id]}" unless source - raise "no snippet" unless doc[:snippet] - - begin - Message.new :source => source, :source_info => doc[:source_info].to_i, - :labels => doc[:label].split(" ").map { |s| s.intern }, - :snippet => doc[:snippet] - rescue MessageFormatError => e - raise IndexError.new(source, "error building message #{doc[:message_id]} at #{source}/#{doc[:source_info]}: #{e.message}") -# rescue StandardError => e -# Message.new_from_index doc, < source, :source_info => doc[:source_info].to_i, + :labels => doc[:label].split(" ").map { |s| s.intern }, + :snippet => doc[:snippet] + rescue MessageFormatError => e + raise IndexError.new(source, "error building message #{doc[:message_id]} at #{source}/#{doc[:source_info]}: #{e.message}") + rescue SourceError => e + nil + end + end + + unless m + fake_header = { + "date" => Time.at(doc[:date].to_i), + "subject" => unwrap_subj(doc[:subject]), + "from" => doc[:from], + "to" => doc[:to], + "message-id" => doc[:message_id], + "references" => doc[:refs], + } + + m = Message.new :labels => doc[:label].split(" ").map { |s| s.intern }, + :snippet => doc[:snippet], :header => fake_header, + :body => < e + raise SourceError, "error parsing message with rmail: #{e.message}" end end end @@ -92,47 +71,40 @@ class Loader end def next - return nil if done? - @dirty = true - start_offset = nil - next_end_offset = @end_offset - - ## @end_offset could be at one of two places here: before a \n and - ## a mbox separator, if it was previously at EOF and a new message - ## was added; or, at the beginning of an mbox separator (in all - ## other cases). + returned_offset = nil + next_offset = cur_offset + @mutex.synchronize do - @f.seek @end_offset - l = @f.gets or return nil - if l =~ /^\s*$/ - start_offset = @f.tell - @f.gets - else - start_offset = @end_offset + @f.seek cur_offset + + ## cur_offset could be at one of two places here: + + ## 1. before a \n and a mbox separator, if it was previously at + ## EOF and a new message was added; or, + ## 2. at the beginning of an mbox separator (in all other + ## cases). + + l = @f.gets or raise "next while at EOF" + if l =~ /^\s*$/ # case 1 + returned_offset = @f.tell + @f.gets # now we're at a BREAK_RE, so skip past it + else # case 2 + returned_offset = cur_offset + ## we've already skipped past the BREAK_RE, to just go end while(line = @f.gets) break if line =~ BREAK_RE - next_end_offset = @f.tell + next_offset = @f.tell end end - @end_offset = next_end_offset - start_offset + self.cur_offset = next_offset + [returned_offset, labels] end - - def each - until @end_offset >= File.size(@f) - n = self.next - yield(n, labels) if n - end - end - - def done?; @end_offset >= File.size(@f); end - def total; File.size @f; end end -Redwood::register_yaml(Loader, %w(filename end_offset usual archived id)) +Redwood::register_yaml(Loader, %w(uri cur_offset usual archived id)) end end diff --git a/lib/sup/message.rb b/lib/sup/message.rb index d9614ba..0f7d6b7 100644 --- a/lib/sup/message.rb +++ b/lib/sup/message.rb @@ -88,8 +88,14 @@ class Message ## if index_entry is specified, will fill in values from that, def initialize opts - @source = opts[:source] - @source_info = opts[:source_info] + if opts[:source] + @source = opts[:source] + @source_info = opts[:source_info] or raise ArgumentError, ":source but no :source_info" + @body = nil + else + @source = @source_info = nil + @body = opts[:body] or raise ArgumentError, "one of :body or :source must be specified" + end @snippet = opts[:snippet] || "" @labels = opts[:labels] || [] @dirty = false @@ -109,7 +115,8 @@ class Message end begin - @date = Time.parse header["date"] + date = header["date"] + @date = (Time === date ? date : Time.parse(header["date"])) rescue ArgumentError => e raise MessageFormatError, "unparsable date #{header['date']}: #{e.message}" end @@ -173,8 +180,11 @@ class Message end def to_chunks - m = @source.load_message @source_info - message_to_chunks m + if @body + [Text.new(@body.split("\n"))] + else + message_to_chunks @source.load_message(@source_info) + end end def raw_header diff --git a/lib/sup/sent.rb b/lib/sup/sent.rb index 1da9adf..1b093d5 100644 --- a/lib/sup/sent.rb +++ b/lib/sup/sent.rb @@ -10,9 +10,9 @@ class SentManager self.class.i_am_the_instance self end - def self.source_name; "sent"; end + def self.source_name; "sent://"; end def self.source_id; 9998; end - def new_source; @source = SentLoader.new @fn; end + def new_source; @source = SentLoader.new; end def write_sent_message date, from_email need_blank = File.exists?(@fn) && !File.zero?(@fn) @@ -30,17 +30,17 @@ class SentManager end class SentLoader < MBox::Loader - def initialize filename, end_offset=0 + def initialize cur_offset=0 + filename = Redwood::SENT_FN File.open(filename, "w") { } unless File.exists? filename - super filename, end_offset, true, true + super "mbox://" + filename, cur_offset, true, true end - def id; SentManager.source_id; end def to_s; SentManager.source_name; end - + def id; SentManager.source_id; end def labels; [:sent, :inbox]; end end -Redwood::register_yaml(SentLoader, %w(filename end_offset)) +Redwood::register_yaml(SentLoader, %w(cur_offset)) end diff --git a/lib/sup/source.rb b/lib/sup/source.rb new file mode 100644 index 0000000..4dc2077 --- /dev/null +++ b/lib/sup/source.rb @@ -0,0 +1,63 @@ +module Redwood + +class SourceError < StandardError; end + +class Source + ## dirty? described whether cur_offset has changed, which means the + ## source needs to be re-saved to disk. + ## + ## broken? means no message can be loaded (e.g. IMAP server is + ## down), so don't even bother. + bool_reader :usual, :archived, :dirty, :broken + attr_reader :cur_offset + attr_accessor :id + + ## You should implement: + ## + ## start_offset + ## end_offset + ## load_header(offset) + ## load_message(offset) + ## raw_header(offset) + ## raw_full_message(offset) + ## next + + def initialize uri, initial_offset=nil, usual=true, archived=false, id=nil + @uri = uri + @cur_offset = initial_offset || start_offset + @usual = usual + @archived = archived + @id = id + @dirty = false + @broken = false + end + + def to_s; @uri; end + def seek_to! o; self.cur_offset = o; end + def reset!; seek_to! start_offset; end + def == o; o.to_s == to_s; end + def done?; cur_offset >= end_offset; end + def is_source_for? s; to_s == s; end + + def each + until done? + n, labels = self.next + raise "no message" unless n + labels += [:inbox] unless archived? + yield n, labels + end + end + +protected + + def cur_offset= o + @cur_offset = o + @dirty = true + end + + attr_writer :broken +end + +Redwood::register_yaml(Source, %w(uri cur_offset usual archived id)) + +end