X-Git-Url: https://git.cworth.org/git?a=blobdiff_plain;f=lib%2Fsup%2Fmessage.rb;h=c06a03fa547a3d6c96dd19945fae0934b907c925;hb=073e1f40024d7721ada08bce7c38d87883ec62f6;hp=4e7d3caf2ca4f3a1b1ace829b3805c2f995356a4;hpb=1e0ead5e3b880ed11c332c0a730c1f06f3ac4f6c;p=sup diff --git a/lib/sup/message.rb b/lib/sup/message.rb index 4e7d3ca..c06a03f 100644 --- a/lib/sup/message.rb +++ b/lib/sup/message.rb @@ -1,5 +1,5 @@ -require 'tempfile' require 'time' +require 'iconv' module Redwood @@ -13,13 +13,13 @@ class MessageFormatError < StandardError; end ## specific module that would detect and link to /ruby-talk:\d+/ ## sequences in the text of an email. (how sweet would that be?) ## -## TODO: integrate with user's addressbook to render names -## appropriately. +## this class cathces all source exceptions. if the underlying source throws +## an error, it is caught and handled. + class Message SNIPPET_LEN = 80 - WRAP_LEN = 80 # wrap at this width RE_PATTERN = /^((re|re[\[\(]\d[\]\)]):\s*)+/i - + ## some utility methods class << self def normalize_subj s; s.gsub(RE_PATTERN, ""); end @@ -27,132 +27,134 @@ class Message def reify_subj s; subj_is_reply?(s) ? s : "Re: " + s; end end - class Attachment - attr_reader :content_type, :desc, :filename - def initialize content_type, desc, part - @content_type = content_type - @desc = desc - @part = part - @file = nil - desc =~ /filename="(.*?)"/ && @filename = $1 - end - - def view! - unless @file - @file = Tempfile.new "redwood.attachment" - @file.print self - @file.close - end - - ## TODO: handle unknown mime-types - system "/usr/bin/run-mailcap --action=view #{@content_type}:#{@file.path}" - $? == 0 - end - - def to_s; @part.decode; end - end - - class Text - attr_reader :lines - def initialize lines - ## do some wrapping - @lines = lines.map { |l| l.chomp.wrap WRAP_LEN }.flatten - end - end - - class Quote - attr_reader :lines - def initialize lines - @lines = lines - end - end - - class Signature - attr_reader :lines - def initialize lines - @lines = lines - end - end - QUOTE_PATTERN = /^\s{0,4}[>|\}]/ BLOCK_QUOTE_PATTERN = /^-----\s*Original Message\s*----+$/ - QUOTE_START_PATTERN = /(^\s*Excerpts from)|(^\s*In message )|(^\s*In article )|(^\s*Quoting )|((wrote|writes|said|says)\s*:\s*$)/ - SIG_PATTERN = /(^-- ?$)|(^\s*----------+\s*$)|(^\s*_________+\s*$)|(^\s*--~--~-)/ + QUOTE_START_PATTERN = /\w.*:$/ + SIG_PATTERN = /(^-- ?$)|(^\s*----------+\s*$)|(^\s*_________+\s*$)|(^\s*--~--~-)|(^\s*--\+\+\*\*==)/ + MAX_SIG_DISTANCE = 15 # lines from the end - DEFAULT_SUBJECT = "(missing subject)" + DEFAULT_SUBJECT = "" DEFAULT_SENDER = "(missing sender)" attr_reader :id, :date, :from, :subj, :refs, :replytos, :to, :source, :cc, :bcc, :labels, :list_address, :recipient_email, :replyto, - :source_info, :chunks + :source_info, :list_subscribe, :list_unsubscribe - bool_reader :dirty, :source_marked_read + bool_reader :dirty, :source_marked_read, :snippet_contains_encrypted_content - ## if you specify a :header, will use values from that. otherwise, will try and - ## load the header from the source. + ## if you specify a :header, will use values from that. otherwise, + ## will try and load the header from the source. def initialize opts @source = opts[:source] or raise ArgumentError, "source can't be nil" @source_info = opts[:source_info] or raise ArgumentError, "source_info can't be nil" - @snippet = opts[:snippet] || "" - @have_snippet = !opts[:snippet].nil? - @labels = opts[:labels] || [] + @snippet = opts[:snippet] + @snippet_contains_encrypted_content = false + @have_snippet = !(opts[:snippet].nil? || opts[:snippet].empty?) + @labels = [] + (opts[:labels] || []) @dirty = false + @encrypted = false @chunks = nil - raise ArgumentError, "need a header" unless opts[:header] - read_header opts[:header] - #read_header(opts[:header] || @source.load_header(@source_info)) + ## we need to initialize this. see comments in parse_header as to + ## why. + @refs = [] + + parse_header(opts[:header] || @source.load_header(@source_info)) end - def read_header header + def parse_header header header.each { |k, v| header[k.downcase] = v } - %w(message-id date).each do |f| - raise MessageFormatError, "no #{f} field in header #{header.inspect} (source #@source offset #@source_info)" unless header.include? f - raise MessageFormatError, "nil #{f} field in header #{header.inspect} (source #@source offset #@source_info)" unless header[f] - end + fakeid = nil + fakename = nil - begin - date = header["date"] - @date = Time === date ? date : Time.parse(header["date"]) - rescue ArgumentError => e - raise MessageFormatError, "unparsable date #{header['date']}: #{e.message}" - end + @id = + if header["message-id"] + sanitize_message_id header["message-id"] + else + fakeid = "sup-faked-" + Digest::MD5.hexdigest(raw_header) + end + + @from = + if header["from"] + PersonManager.person_for header["from"] + else + fakename = "Sup Auto-generated Fake Sender " + PersonManager.person_for fakename + end + + Redwood::log "faking message-id for message from #@from: #{id}" if fakeid + Redwood::log "faking from for message #@id: #{fakename}" if fakename + + date = header["date"] + @date = + case date + when Time + date + when String + begin + Time.parse date + rescue ArgumentError => e + Redwood::log "faking date header for #{@id} due to error parsing date #{header['date'].inspect}: #{e.message}" + Time.now + end + else + Redwood::log "faking date header for #{@id}" + Time.now + end @subj = header.member?("subject") ? header["subject"].gsub(/\s+/, " ").gsub(/\s+$/, "") : DEFAULT_SUBJECT - @from = Person.for header["from"] - @to = Person.for_several header["to"] - @cc = Person.for_several header["cc"] - @bcc = Person.for_several header["bcc"] - @id = header["message-id"] - @refs = (header["references"] || "").gsub(/[<>]/, "").split(/\s+/).flatten - @replytos = (header["in-reply-to"] || "").scan(/<(.*?)>/).flatten - @replyto = Person.for header["reply-to"] + @to = PersonManager.people_for header["to"] + @cc = PersonManager.people_for header["cc"] + @bcc = PersonManager.people_for header["bcc"] + + ## before loading our full header from the source, we can actually + ## have some extra refs set by the UI. (this happens when the user + ## joins threads manually). so we will merge the current refs values + ## in here. + refs = (header["references"] || "").scan(/<(.+?)>/).map { |x| sanitize_message_id x.first } + @refs = (@refs + refs).uniq + @replytos = (header["in-reply-to"] || "").scan(/<(.+?)>/).map { |x| sanitize_message_id x.first } + + @replyto = PersonManager.person_for header["reply-to"] @list_address = if header["list-post"] - @list_address = Person.for header["list-post"].gsub(/^$/, "") + @list_address = PersonManager.person_for header["list-post"].gsub(/^$/, "") else nil end @recipient_email = header["envelope-to"] || header["x-original-to"] || header["delivered-to"] @source_marked_read = header["status"] == "RO" + @list_subscribe = header["list-subscribe"] + @list_unsubscribe = header["list-unsubscribe"] + end + private :parse_header + + def add_ref ref + @refs << ref + @dirty = true + end + + def remove_ref ref + @dirty = true if @refs.delete ref end - private :read_header - def broken?; @source.broken?; end - def snippet; @snippet || chunks && @snippet; end + def snippet; @snippet || (chunks && @snippet); end def is_list_message?; !@list_address.nil?; end - def is_draft?; DraftLoader === @source; end + def is_draft?; @source.is_a? DraftLoader; end def draft_filename raise "not a draft" unless is_draft? @source.fn_for_offset @source_info end + def sanitize_message_id mid; mid.gsub(/\s/, "") end + def save index - return if broken? - index.update_message self if @dirty + return unless @dirty + index.sync_message self @dirty = false + true end def has_label? t; @labels.member? t; end @@ -176,11 +178,16 @@ class Message @dirty = true end + def chunks + load_from_source! + @chunks + end + ## this is called when the message body needs to actually be loaded. def load_from_source! @chunks ||= - if @source.broken? - [Text.new(error_message(@source.broken_msg.split("\n")))] + if @source.has_errors? + [Chunk::Text.new(error_message(@source.error.message).split("\n"))] else begin ## we need to re-read the header because it contains information @@ -191,10 +198,15 @@ class Message ## bloat the index. ## actually, it's also the differentiation between to/cc/bcc, ## so i will keep this. - read_header @source.load_header(@source_info) + parse_header @source.load_header(@source_info) message_to_chunks @source.load_message(@source_info) rescue SourceError, SocketError, MessageFormatError => e - [Text.new(error_message(e.message))] + Redwood::log "problem getting messages from #{@source}: #{e.message}" + ## we need force_to_top here otherwise this window will cover + ## up the error message one + @source.error ||= e + Redwood::report_broken_sources :force_to_top => true + [Chunk::Text.new(error_message(e.message).split("\n"))] end end end @@ -204,8 +216,13 @@ class Message #@snippet... *********************************************************************** -* An error occurred while loading this message. It is possible that * -* the source has changed, or (in the case of remote sources) is down. * + An error occurred while loading this message. It is possible that + the source has changed, or (in the case of remote sources) is down. + You can check the log for errors, though hopefully an error window + should have popped up at some point. + + The message location was: + #@source##@source_info *********************************************************************** The error message was: @@ -213,38 +230,49 @@ The error message was: EOS end - def raw_header + ## wrap any source methods that might throw sourceerrors + def with_source_errors_handled begin - @source.raw_header @source_info + yield rescue SourceError => e + Redwood::log "problem getting messages from #{@source}: #{e.message}" + @source.error ||= e + Redwood::report_broken_sources :force_to_top => true error_message e.message end end - def raw_full_message - begin - @source.raw_full_message @source_info - rescue SourceError => e - error_message(e.message) - end + def raw_header + with_source_errors_handled { @source.raw_header @source_info } + end + + def raw_message + with_source_errors_handled { @source.raw_message @source_info } end - def content + ## much faster than raw_message + def each_raw_message_line &b + with_source_errors_handled { @source.each_raw_message_line(@source_info, &b) } + end + + ## returns all the content from a message that will be indexed + def indexable_content + load_from_source! [ - from && "#{from.name} #{from.email}", - to.map { |p| "#{p.name} #{p.email}" }, - cc.map { |p| "#{p.name} #{p.email}" }, - bcc.map { |p| "#{p.name} #{p.email}" }, - chunks.select { |c| c.is_a? Text }.map { |c| c.lines }, + from && from.indexable_content, + to.map { |p| p.indexable_content }, + cc.map { |p| p.indexable_content }, + bcc.map { |p| p.indexable_content }, + chunks.select { |c| c.is_a? Chunk::Text }.map { |c| c.lines }, Message.normalize_subj(subj), ].flatten.compact.join " " end - def basic_body_lines - chunks.find_all { |c| c.is_a?(Text) || c.is_a?(Quote) }.map { |c| c.lines }.flatten + def quotable_body_lines + chunks.find_all { |c| c.quotable? }.map { |c| c.lines }.flatten end - def basic_header_lines + def quotable_header_lines ["From: #{@from.full_address}"] + (@to.empty? ? [] : ["To: " + @to.map { |p| p.full_address }.join(", ")]) + (@cc.empty? ? [] : ["Cc: " + @cc.map { |p| p.full_address }.join(", ")]) + @@ -255,28 +283,155 @@ EOS private - ## everything RubyMail-specific goes here. - def message_to_chunks m - ret = [] << - case m.header.content_type - when "text/plain", nil - m.body && body = m.decode or raise MessageFormatError, "for some bizarre reason, RubyMail was unable to parse this message." - text_to_chunks body.normalize_whitespace.split("\n") - when /^multipart\// - nil + ## here's where we handle decoding mime attachments. unfortunately + ## but unsurprisingly, the world of mime attachments is a bit of a + ## mess. as an empiricist, i'm basing the following behavior on + ## observed mail rather than on interpretations of rfcs, so probably + ## this will have to be tweaked. + ## + ## the general behavior i want is: ignore content-disposition, at + ## least in so far as it suggests something being inline vs being an + ## attachment. (because really, that should be the recipient's + ## decision to make.) if a mime part is text/plain, OR if the user + ## decoding hook converts it, then decode it and display it + ## inline. for these decoded attachments, if it has associated + ## filename, then make it collapsable and individually saveable; + ## otherwise, treat it as regular body text. + ## + ## everything else is just an attachment and is not displayed + ## inline. + ## + ## so, in contrast to mutt, the user is not exposed to the workings + ## of the gruesome slaughterhouse and sausage factory that is a + ## mime-encoded message, but need only see the delicious end + ## product. + + def multipart_signed_to_chunks m + if m.body.size != 2 + Redwood::log "warning: multipart/signed with #{m.body.size} parts (expecting 2)" + return + end + + payload, signature = m.body + if signature.multipart? + Redwood::log "warning: multipart/signed with payload multipart #{payload.multipart?} and signature multipart #{signature.multipart?}" + return + end + + ## this probably will never happen + if payload.header.content_type == "application/pgp-signature" + Redwood::log "warning: multipart/signed with payload content type #{payload.header.content_type}" + return + end + + if signature.header.content_type != "application/pgp-signature" + ## unknown signature type; just ignore. + #Redwood::log "warning: multipart/signed with signature content type #{signature.header.content_type}" + return + end + + [CryptoManager.verify(payload, signature), message_to_chunks(payload)].flatten.compact + end + + def multipart_encrypted_to_chunks m + if m.body.size != 2 + Redwood::log "warning: multipart/encrypted with #{m.body.size} parts (expecting 2)" + return + end + + control, payload = m.body + if control.multipart? + Redwood::log "warning: multipart/encrypted with control multipart #{control.multipart?} and payload multipart #{payload.multipart?}" + return + end + + if payload.header.content_type != "application/octet-stream" + Redwood::log "warning: multipart/encrypted with payload content type #{payload.header.content_type}" + return + end + + if control.header.content_type != "application/pgp-encrypted" + Redwood::log "warning: multipart/encrypted with control content type #{signature.header.content_type}" + return + end + + decryptedm, sig, notice = CryptoManager.decrypt payload + children = message_to_chunks(decryptedm, true) if decryptedm + [notice, sig, children].flatten.compact + end + + def message_to_chunks m, encrypted=false, sibling_types=[] + if m.multipart? + chunks = + case m.header.content_type + when "multipart/signed" + multipart_signed_to_chunks m + when "multipart/encrypted" + multipart_encrypted_to_chunks m + end + + unless chunks + sibling_types = m.body.map { |p| p.header.content_type } + chunks = m.body.map { |p| message_to_chunks p, encrypted, sibling_types }.flatten.compact + end + + chunks + elsif m.header.content_type == "message/rfc822" + payload = RMail::Parser.read(m.body) + from = payload.header.from.first + from_person = from ? PersonManager.person_for(from.format) : nil + [Chunk::EnclosedMessage.new(from_person, payload.to_s)] + else + filename = + ## first, paw through the headers looking for a filename + if m.header["Content-Disposition"] && m.header["Content-Disposition"] =~ /filename="?(.*?[^\\])("|;|$)/ + $1 + elsif m.header["Content-Type"] && m.header["Content-Type"] =~ /name="?(.*?[^\\])("|;|$)/ + $1 + + ## haven't found one, but it's a non-text message. fake + ## it. + ## + ## TODO: make this less lame. + elsif m.header["Content-Type"] && m.header["Content-Type"] !~ /^text\/plain/ + extension = + case m.header["Content-Type"] + when /text\/html/: "html" + when /image\/(.*)/: $1 + end + + ["sup-attachment-#{Time.now.to_i}-#{rand 10000}", extension].join(".") + end + + ## if there's a filename, we'll treat it as an attachment. + if filename + [Chunk::Attachment.new(m.header.content_type, filename, m, sibling_types)] + + ## otherwise, it's body text else - disp = m.header["Content-Disposition"] || "" - Attachment.new m.header.content_type, disp.gsub(/[\s\n]+/, " "), m + body = Message.convert_from m.decode, m.charset if m.body + text_to_chunks((body || "").normalize_whitespace.split("\n"), encrypted) end - - m.each_part { |p| ret << message_to_chunks(p) } if m.multipart? - ret.compact.flatten + end + end + + def self.convert_from body, charset + charset = "utf-8" if charset =~ /UTF_?8/i + begin + raise MessageFormatError, "RubyMail decode returned a null body" unless body + return body unless charset + Iconv.iconv($encoding + "//IGNORE", charset, body + " ").join[0 .. -2] + rescue Errno::EINVAL, Iconv::InvalidEncoding, Iconv::IllegalSequence, MessageFormatError => e + Redwood::log "warning: error (#{e.class.name}) decoding message body from #{charset}: #{e.message}" + File.open("sup-unable-to-decode.txt", "w") { |f| f.write body } + body + end end ## parse the lines of text into chunk objects. the heuristics here ## need tweaking in some nice manner. TODO: move these heuristics ## into the classes themselves. - def text_to_chunks lines + def text_to_chunks lines, encrypted state = :text # one of :text, :quote, or :sig chunks = [] chunk_lines = [] @@ -288,7 +443,7 @@ private when :text newstate = nil - if line =~ QUOTE_PATTERN || (line =~ QUOTE_START_PATTERN && (nextline =~ QUOTE_PATTERN || nextline =~ QUOTE_START_PATTERN)) + if line =~ QUOTE_PATTERN || (line =~ QUOTE_START_PATTERN && nextline =~ QUOTE_PATTERN) newstate = :quote elsif line =~ SIG_PATTERN && (lines.length - i) < MAX_SIG_DISTANCE newstate = :sig @@ -297,7 +452,7 @@ private end if newstate - chunks << Text.new(chunk_lines) unless chunk_lines.empty? + chunks << Chunk::Text.new(chunk_lines) unless chunk_lines.empty? chunk_lines = [line] state = newstate else @@ -307,7 +462,7 @@ private when :quote newstate = nil - if line =~ QUOTE_PATTERN || line =~ QUOTE_START_PATTERN #|| line =~ /^\s*$/ + if line =~ QUOTE_PATTERN || (line =~ /^\s*$/ && nextline =~ QUOTE_PATTERN) chunk_lines << line elsif line =~ SIG_PATTERN && (lines.length - i) < MAX_SIG_DISTANCE newstate = :sig @@ -318,37 +473,35 @@ private if newstate if chunk_lines.empty? # nothing - elsif chunk_lines.size == 1 - chunks << Text.new(chunk_lines) # forget about one-line quotes else - chunks << Quote.new(chunk_lines) + chunks << Chunk::Quote.new(chunk_lines) end chunk_lines = [line] state = newstate end - when :block_quote - chunk_lines << line - - when :sig + when :block_quote, :sig chunk_lines << line end - + if !@have_snippet && state == :text && (@snippet.nil? || @snippet.length < SNIPPET_LEN) && line !~ /[=\*#_-]{3,}/ && line !~ /^\s*$/ + @snippet ||= "" @snippet += " " unless @snippet.empty? @snippet += line.gsub(/^\s+/, "").gsub(/[\r\n]/, "").gsub(/\s+/, " ") @snippet = @snippet[0 ... SNIPPET_LEN].chomp + @dirty = true unless encrypted && $config[:discard_snippets_from_encrypted_messages] + @snippet_contains_encrypted_content = true if encrypted end end ## final object case state when :quote, :block_quote - chunks << Quote.new(chunk_lines) unless chunk_lines.empty? + chunks << Chunk::Quote.new(chunk_lines) unless chunk_lines.empty? when :text - chunks << Text.new(chunk_lines) unless chunk_lines.empty? + chunks << Chunk::Text.new(chunk_lines) unless chunk_lines.empty? when :sig - chunks << Signature.new(chunk_lines) unless chunk_lines.empty? + chunks << Chunk::Signature.new(chunk_lines) unless chunk_lines.empty? end chunks end