X-Git-Url: https://git.cworth.org/git?a=blobdiff_plain;f=lib%2Fsup%2Fmessage.rb;h=853f2fccb3676d4df05759d4b3e81e19df04efd4;hb=f7b27c55c1be05de3d7952f6a7cf630142e0bbcd;hp=812d743dfca7d675ac7e2d8b1ab2b502ffc35257;hpb=d2f2caef9b51e38bc5e9501db7fd1fcd84b36b46;p=sup diff --git a/lib/sup/message.rb b/lib/sup/message.rb index 812d743..853f2fc 100644 --- a/lib/sup/message.rb +++ b/lib/sup/message.rb @@ -1,4 +1,3 @@ -require 'tempfile' require 'time' require 'iconv' @@ -13,25 +12,14 @@ class MessageFormatError < StandardError; end ## i would like, for example, to be able to add in a ruby-talk ## specific module that would detect and link to /ruby-talk:\d+/ ## sequences in the text of an email. (how sweet would that be?) +## +## this class cathces all source exceptions. if the underlying source throws +## an error, it is caught and handled. + class Message SNIPPET_LEN = 80 - WRAP_LEN = 80 # wrap at this width RE_PATTERN = /^((re|re[\[\(]\d[\]\)]):\s*)+/i - HookManager.register "mime-decode", < content_type, - :filename => lambda { write_to_disk }, :sibling_types => sibling_types - @lines = text.split("\n") if text - end - end - - def inlineable?; !@lines.nil? end - - def view! - path = write_to_disk - system "/usr/bin/run-mailcap --action=view #{@content_type}:#{path} >& /dev/null" - $? == 0 - end - - private - - def write_to_disk - file = Tempfile.new "redwood.attachment" - file.print @raw_content - file.close - file.path - end - end - - class Text - attr_reader :lines - def initialize lines - ## do some wrapping - @lines = lines.map { |l| l.chomp.wrap WRAP_LEN }.flatten - end - end - - class Quote - attr_reader :lines - def initialize lines - @lines = lines - end - end - - class Signature - attr_reader :lines - def initialize lines - @lines = lines - end - end - QUOTE_PATTERN = /^\s{0,4}[>|\}]/ BLOCK_QUOTE_PATTERN = /^-----\s*Original Message\s*----+$/ - QUOTE_START_PATTERN = /(^\s*Excerpts from)|(^\s*In message )|(^\s*In article )|(^\s*Quoting )|((wrote|writes|said|says)\s*:\s*$)/ - SIG_PATTERN = /(^-- ?$)|(^\s*----------+\s*$)|(^\s*_________+\s*$)|(^\s*--~--~-)/ + QUOTE_START_PATTERN = /\w.*:$/ + SIG_PATTERN = /(^-- ?$)|(^\s*----------+\s*$)|(^\s*_________+\s*$)|(^\s*--~--~-)|(^\s*--\+\+\*\*==)/ MAX_SIG_DISTANCE = 15 # lines from the end DEFAULT_SUBJECT = "" @@ -115,34 +38,53 @@ EOS attr_reader :id, :date, :from, :subj, :refs, :replytos, :to, :source, :cc, :bcc, :labels, :list_address, :recipient_email, :replyto, - :source_info, :chunks + :source_info, :list_subscribe, :list_unsubscribe - bool_reader :dirty, :source_marked_read + bool_reader :dirty, :source_marked_read, :snippet_contains_encrypted_content ## if you specify a :header, will use values from that. otherwise, ## will try and load the header from the source. def initialize opts @source = opts[:source] or raise ArgumentError, "source can't be nil" @source_info = opts[:source_info] or raise ArgumentError, "source_info can't be nil" - @snippet = opts[:snippet] || "" - @have_snippet = !opts[:snippet].nil? + @snippet = opts[:snippet] + @snippet_contains_encrypted_content = false + @have_snippet = !(opts[:snippet].nil? || opts[:snippet].empty?) @labels = [] + (opts[:labels] || []) @dirty = false + @encrypted = false @chunks = nil + ## we need to initialize this. see comments in parse_header as to + ## why. + @refs = [] + parse_header(opts[:header] || @source.load_header(@source_info)) end def parse_header header header.each { |k, v| header[k.downcase] = v } - @from = PersonManager.person_for header["from"] + fakeid = nil + fakename = nil - @id = header["message-id"] - unless @id - @id = "sup-faked-" + Digest::MD5.hexdigest(raw_header) - Redwood::log "faking message-id for message from #@from: #@id" - end + @id = + if header["message-id"] + sanitize_message_id header["message-id"] + else + fakeid = "sup-faked-" + Digest::MD5.hexdigest(raw_header) + end + + @from = + if header["from"] + PersonManager.person_for header["from"] + else + fakename = "Sup Auto-generated Fake Sender " + PersonManager.person_for fakename + end + + Redwood::log "faking message-id for message from #@from: #{id}" if fakeid + Redwood::log "faking from for message #@id: #{fakename}" if fakename date = header["date"] @date = @@ -164,8 +106,15 @@ EOS @to = PersonManager.people_for header["to"] @cc = PersonManager.people_for header["cc"] @bcc = PersonManager.people_for header["bcc"] - @refs = (header["references"] || "").gsub(/[<>]/, "").split(/\s+/).flatten - @replytos = (header["in-reply-to"] || "").scan(/<(.*?)>/).flatten + + ## before loading our full header from the source, we can actually + ## have some extra refs set by the UI. (this happens when the user + ## joins threads manually). so we will merge the current refs values + ## in here. + refs = (header["references"] || "").scan(/<(.+?)>/).map { |x| sanitize_message_id x.first } + @refs = (@refs + refs).uniq + @replytos = (header["in-reply-to"] || "").scan(/<(.+?)>/).map { |x| sanitize_message_id x.first } + @replyto = PersonManager.person_for header["reply-to"] @list_address = if header["list-post"] @@ -176,10 +125,17 @@ EOS @recipient_email = header["envelope-to"] || header["x-original-to"] || header["delivered-to"] @source_marked_read = header["status"] == "RO" + @list_subscribe = header["list-subscribe"] + @list_unsubscribe = header["list-unsubscribe"] end private :parse_header - def snippet; @snippet || chunks && @snippet; end + def add_ref ref + @refs << ref + @dirty = true + end + + def snippet; @snippet || (chunks && @snippet); end def is_list_message?; !@list_address.nil?; end def is_draft?; @source.is_a? DraftLoader; end def draft_filename @@ -187,6 +143,8 @@ EOS @source.fn_for_offset @source_info end + def sanitize_message_id mid; mid.gsub(/\s/, "") end + def save index index.sync_message self if @dirty @dirty = false @@ -213,11 +171,16 @@ EOS @dirty = true end + def chunks + load_from_source! + @chunks + end + ## this is called when the message body needs to actually be loaded. def load_from_source! @chunks ||= if @source.has_errors? - [Text.new(error_message(@source.error.message.split("\n")))] + [Chunk::Text.new(error_message(@source.error.message).split("\n"))] else begin ## we need to re-read the header because it contains information @@ -234,8 +197,9 @@ EOS Redwood::log "problem getting messages from #{@source}: #{e.message}" ## we need force_to_top here otherwise this window will cover ## up the error message one + @source.error ||= e Redwood::report_broken_sources :force_to_top => true - [Text.new(error_message(e.message))] + [Chunk::Text.new(error_message(e.message).split("\n"))] end end end @@ -259,22 +223,29 @@ The error message was: EOS end - def raw_header + ## wrap any source methods that might throw sourceerrors + def with_source_errors_handled begin - @source.raw_header @source_info + yield rescue SourceError => e Redwood::log "problem getting messages from #{@source}: #{e.message}" + @source.error ||= e + Redwood::report_broken_sources :force_to_top => true error_message e.message end end - def raw_full_message - begin - @source.raw_full_message @source_info - rescue SourceError => e - Redwood::log "problem getting messages from #{@source}: #{e.message}" - error_message(e.message) - end + def raw_header + with_source_errors_handled { @source.raw_header @source_info } + end + + def raw_message + with_source_errors_handled { @source.raw_message @source_info } + end + + ## much faster than raw_message + def each_raw_message_line &b + with_source_errors_handled { @source.each_raw_message_line(@source_info, &b) } end def content @@ -284,16 +255,16 @@ EOS to.map { |p| "#{p.name} #{p.email}" }, cc.map { |p| "#{p.name} #{p.email}" }, bcc.map { |p| "#{p.name} #{p.email}" }, - chunks.select { |c| c.is_a? Text }.map { |c| c.lines }, + chunks.select { |c| c.is_a? Chunk::Text }.map { |c| c.lines }, Message.normalize_subj(subj), ].flatten.compact.join " " end - def basic_body_lines - chunks.find_all { |c| c.is_a?(Text) || c.is_a?(Quote) }.map { |c| c.lines }.flatten + def quotable_body_lines + chunks.find_all { |c| c.quotable? }.map { |c| c.lines }.flatten end - def basic_header_lines + def quotable_header_lines ["From: #{@from.full_address}"] + (@to.empty? ? [] : ["To: " + @to.map { |p| p.full_address }.join(", ")]) + (@cc.empty? ? [] : ["Cc: " + @cc.map { |p| p.full_address }.join(", ")]) + @@ -326,44 +297,123 @@ private ## of the gruesome slaughterhouse and sausage factory that is a ## mime-encoded message, but need only see the delicious end ## product. - def message_to_chunks m, sibling_types=[] + + def multipart_signed_to_chunks m + if m.body.size != 2 + Redwood::log "warning: multipart/signed with #{m.body.size} parts (expecting 2)" + return + end + + payload, signature = m.body + if signature.multipart? + Redwood::log "warning: multipart/signed with payload multipart #{payload.multipart?} and signature multipart #{signature.multipart?}" + return + end + + ## this probably will never happen + if payload.header.content_type == "application/pgp-signature" + Redwood::log "warning: multipart/signed with payload content type #{payload.header.content_type}" + return + end + + if signature.header.content_type != "application/pgp-signature" + ## unknown signature type; just ignore. + #Redwood::log "warning: multipart/signed with signature content type #{signature.header.content_type}" + return + end + + [CryptoManager.verify(payload, signature), message_to_chunks(payload)].flatten.compact + end + + def multipart_encrypted_to_chunks m + if m.body.size != 2 + Redwood::log "warning: multipart/encrypted with #{m.body.size} parts (expecting 2)" + return + end + + control, payload = m.body + if control.multipart? + Redwood::log "warning: multipart/encrypted with control multipart #{control.multipart?} and payload multipart #{payload.multipart?}" + return + end + + if payload.header.content_type != "application/octet-stream" + Redwood::log "warning: multipart/encrypted with payload content type #{payload.header.content_type}" + return + end + + if control.header.content_type != "application/pgp-encrypted" + Redwood::log "warning: multipart/encrypted with control content type #{signature.header.content_type}" + return + end + + decryptedm, sig, notice = CryptoManager.decrypt payload + children = message_to_chunks(decryptedm, true) if decryptedm + [notice, sig, children].flatten.compact + end + + def message_to_chunks m, encrypted=false, sibling_types=[] if m.multipart? - sibling_types = m.body.map { |p| p.header.content_type } - m.body.map { |p| message_to_chunks p, sibling_types }.flatten.compact # recurse + chunks = + case m.header.content_type + when "multipart/signed" + multipart_signed_to_chunks m + when "multipart/encrypted" + multipart_encrypted_to_chunks m + end + + unless chunks + sibling_types = m.body.map { |p| p.header.content_type } + chunks = m.body.map { |p| message_to_chunks p, encrypted, sibling_types }.flatten.compact + end + + chunks + elsif m.header.content_type == "message/rfc822" + payload = RMail::Parser.read(m.body) + from = payload.header.from.first + from_person = from ? PersonManager.person_for(from.format) : nil + [Chunk::EnclosedMessage.new(from_person, payload.to_s)] else filename = ## first, paw through the headers looking for a filename - if m.header["Content-Disposition"] && - m.header["Content-Disposition"] =~ /filename="?(.*?[^\\])("|;|$)/ + if m.header["Content-Disposition"] && m.header["Content-Disposition"] =~ /filename="?(.*?[^\\])("|;|$)/ $1 - elsif m.header["Content-Type"] && - m.header["Content-Type"] =~ /name=(.*?)(;|$)/ + elsif m.header["Content-Type"] && m.header["Content-Type"] =~ /name="?(.*?[^\\])("|;|$)/ $1 ## haven't found one, but it's a non-text message. fake ## it. + ## + ## TODO: make this less lame. elsif m.header["Content-Type"] && m.header["Content-Type"] !~ /^text\/plain/ - "sup-attachment-#{Time.now.to_i}-#{rand 10000}" + extension = + case m.header["Content-Type"] + when /text\/html/: "html" + when /image\/(.*)/: $1 + end + + ["sup-attachment-#{Time.now.to_i}-#{rand 10000}", extension].join(".") end ## if there's a filename, we'll treat it as an attachment. if filename - [Attachment.new(m.header.content_type, filename, m, sibling_types)] + [Chunk::Attachment.new(m.header.content_type, filename, m, sibling_types)] ## otherwise, it's body text else - body = Message.convert_from m.body, m.charset - text_to_chunks body.normalize_whitespace.split("\n") + body = Message.convert_from m.decode, m.charset if m.body + text_to_chunks (body || "").normalize_whitespace.split("\n"), encrypted end end end def self.convert_from body, charset - return body unless charset - + charset = "utf-8" if charset =~ /UTF_?8/i begin - Iconv.iconv($encoding, charset, body).join - rescue Errno::EINVAL, Iconv::InvalidEncoding, Iconv::IllegalSequence => e + raise MessageFormatError, "RubyMail decode returned a null body" unless body + return body unless charset + Iconv.iconv($encoding + "//IGNORE", charset, body + " ").join[0 .. -2] + rescue Errno::EINVAL, Iconv::InvalidEncoding, Iconv::IllegalSequence, MessageFormatError => e Redwood::log "warning: error (#{e.class.name}) decoding message body from #{charset}: #{e.message}" File.open("sup-unable-to-decode.txt", "w") { |f| f.write body } body @@ -373,7 +423,7 @@ private ## parse the lines of text into chunk objects. the heuristics here ## need tweaking in some nice manner. TODO: move these heuristics ## into the classes themselves. - def text_to_chunks lines + def text_to_chunks lines, encrypted state = :text # one of :text, :quote, or :sig chunks = [] chunk_lines = [] @@ -385,7 +435,7 @@ private when :text newstate = nil - if line =~ QUOTE_PATTERN || (line =~ QUOTE_START_PATTERN && (nextline =~ QUOTE_PATTERN || nextline =~ QUOTE_START_PATTERN)) + if line =~ QUOTE_PATTERN || (line =~ QUOTE_START_PATTERN && nextline =~ QUOTE_PATTERN) newstate = :quote elsif line =~ SIG_PATTERN && (lines.length - i) < MAX_SIG_DISTANCE newstate = :sig @@ -394,7 +444,7 @@ private end if newstate - chunks << Text.new(chunk_lines) unless chunk_lines.empty? + chunks << Chunk::Text.new(chunk_lines) unless chunk_lines.empty? chunk_lines = [line] state = newstate else @@ -404,7 +454,7 @@ private when :quote newstate = nil - if line =~ QUOTE_PATTERN || line =~ QUOTE_START_PATTERN #|| line =~ /^\s*$/ + if line =~ QUOTE_PATTERN || (line =~ /^\s*$/ && nextline =~ QUOTE_PATTERN) chunk_lines << line elsif line =~ SIG_PATTERN && (lines.length - i) < MAX_SIG_DISTANCE newstate = :sig @@ -416,7 +466,7 @@ private if chunk_lines.empty? # nothing else - chunks << Quote.new(chunk_lines) + chunks << Chunk::Quote.new(chunk_lines) end chunk_lines = [line] state = newstate @@ -425,22 +475,25 @@ private when :block_quote, :sig chunk_lines << line end - + if !@have_snippet && state == :text && (@snippet.nil? || @snippet.length < SNIPPET_LEN) && line !~ /[=\*#_-]{3,}/ && line !~ /^\s*$/ + @snippet ||= "" @snippet += " " unless @snippet.empty? @snippet += line.gsub(/^\s+/, "").gsub(/[\r\n]/, "").gsub(/\s+/, " ") @snippet = @snippet[0 ... SNIPPET_LEN].chomp + @dirty = true unless encrypted && $config[:discard_snippets_from_encrypted_messages] + @snippet_contains_encrypted_content = true if encrypted end end ## final object case state when :quote, :block_quote - chunks << Quote.new(chunk_lines) unless chunk_lines.empty? + chunks << Chunk::Quote.new(chunk_lines) unless chunk_lines.empty? when :text - chunks << Text.new(chunk_lines) unless chunk_lines.empty? + chunks << Chunk::Text.new(chunk_lines) unless chunk_lines.empty? when :sig - chunks << Signature.new(chunk_lines) unless chunk_lines.empty? + chunks << Chunk::Signature.new(chunk_lines) unless chunk_lines.empty? end chunks end