6 class MessageFormatError < StandardError; end
8 ## a Message is what's threaded.
10 ## it is also where the parsing for quotes and signatures is done, but
11 ## that should be moved out to a separate class at some point (because
12 ## i would like, for example, to be able to add in a ruby-talk
13 ## specific module that would detect and link to /ruby-talk:\d+/
14 ## sequences in the text of an email. (how sweet would that be?)
16 ## this class catches all source exceptions. if the underlying source
17 ## throws an error, it is caught and handled.
21 RE_PATTERN = /^((re|re[\[\(]\d[\]\)]):\s*)+/i
23 ## some utility methods
25 def normalize_subj s; s.gsub(RE_PATTERN, ""); end
26 def subj_is_reply? s; s =~ RE_PATTERN; end
27 def reify_subj s; subj_is_reply?(s) ? s : "Re: " + s; end
30 QUOTE_PATTERN = /^\s{0,4}[>|\}]/
31 BLOCK_QUOTE_PATTERN = /^-----\s*Original Message\s*----+$/
32 QUOTE_START_PATTERN = /\w.*:$/
33 SIG_PATTERN = /(^-- ?$)|(^\s*----------+\s*$)|(^\s*_________+\s*$)|(^\s*--~--~-)|(^\s*--\+\+\*\*==)/
35 MAX_SIG_DISTANCE = 15 # lines from the end
37 DEFAULT_SENDER = "(missing sender)"
39 attr_reader :id, :date, :from, :subj, :refs, :replytos, :to, :source,
40 :cc, :bcc, :labels, :attachments, :list_address, :recipient_email, :replyto,
41 :source_info, :list_subscribe, :list_unsubscribe
43 bool_reader :dirty, :source_marked_read, :snippet_contains_encrypted_content
45 ## if you specify a :header, will use values from that. otherwise,
46 ## will try and load the header from the source.
48 @source = opts[:source] or raise ArgumentError, "source can't be nil"
49 @source_info = opts[:source_info] or raise ArgumentError, "source_info can't be nil"
50 @snippet = opts[:snippet]
51 @snippet_contains_encrypted_content = false
52 @have_snippet = !(opts[:snippet].nil? || opts[:snippet].empty?)
53 @labels = (opts[:labels] || []).to_set_of_symbols
59 ## we need to initialize this. see comments in parse_header as to
63 #parse_header(opts[:header] || @source.load_header(@source_info))
66 def parse_header header
67 @id = if header["message-id"]
68 mid = header["message-id"] =~ /<(.+?)>/ ? $1 : header["message-id"]
69 sanitize_message_id mid
71 id = "sup-faked-" + Digest::MD5.hexdigest(raw_header)
73 #Redwood::log "faking non-existent message-id for message from #{from}: #{id}"
77 @from = Person.from_address(if header["from"]
80 name = "Sup Auto-generated Fake Sender <sup@fake.sender.example.com>"
81 #Redwood::log "faking non-existent sender for message #@id: #{name}"
85 @date = case(date = header["date"])
91 rescue ArgumentError => e
92 #Redwood::log "faking mangled date header for #{@id} (orig #{header['date'].inspect} gave error: #{e.message})"
96 #Redwood::log "faking non-existent date header for #{@id}"
100 @subj = header.member?("subject") ? header["subject"].gsub(/\s+/, " ").gsub(/\s+$/, "") : DEFAULT_SUBJECT
101 @to = Person.from_address_list header["to"]
102 @cc = Person.from_address_list header["cc"]
103 @bcc = Person.from_address_list header["bcc"]
105 ## before loading our full header from the source, we can actually
106 ## have some extra refs set by the UI. (this happens when the user
107 ## joins threads manually). so we will merge the current refs values
109 refs = (header["references"] || "").scan(/<(.+?)>/).map { |x| sanitize_message_id x.first }
110 @refs = (@refs + refs).uniq
111 @replytos = (header["in-reply-to"] || "").scan(/<(.+?)>/).map { |x| sanitize_message_id x.first }
113 @replyto = Person.from_address header["reply-to"]
115 if header["list-post"]
116 @list_address = Person.from_address header["list-post"].gsub(/^<mailto:|>$/, "")
121 @recipient_email = header["envelope-to"] || header["x-original-to"] || header["delivered-to"]
122 @source_marked_read = header["status"] == "RO"
123 @list_subscribe = header["list-subscribe"]
124 @list_unsubscribe = header["list-unsubscribe"]
133 @dirty = true if @refs.delete ref
136 def snippet; @snippet || (chunks && @snippet); end
137 def is_list_message?; !@list_address.nil?; end
138 def is_draft?; @source.is_a? DraftLoader; end
140 raise "not a draft" unless is_draft?
141 @source.fn_for_offset @source_info
144 ## sanitize message ids by removing spaces and non-ascii characters.
145 ## also, truncate to 255 characters. all these steps are necessary
146 ## to make ferret happy. of course, we probably fuck up a couple
147 ## valid message ids as well. as long as we're consistent, this
148 ## should be fine, though.
150 ## also, mostly the message ids that are changed by this belong to
153 ## an alternative would be to SHA1 or MD5 all message ids on a regular basis.
155 def sanitize_message_id mid; mid.gsub(/(\s|[^\000-\177])+/, "")[0..254] end
159 index.sync_message self
164 def has_label? t; @labels.member? t; end
166 return if @labels.member? t
167 @labels = (@labels + [t]).to_set_of_symbols
171 return unless @labels.member? t
181 @labels = l.to_set_of_symbols
190 ## this is called when the message body needs to actually be loaded.
191 def load_from_source!
193 if @source.respond_to?(:has_errors?) && @source.has_errors?
194 [Chunk::Text.new(error_message(@source.error.message).split("\n"))]
197 ## we need to re-read the header because it contains information
198 ## that we don't store in the index. actually i think it's just
199 ## the mailing list address (if any), so this is kinda overkill.
200 ## i could just store that in the index, but i think there might
201 ## be other things like that in the future, and i'd rather not
203 ## actually, it's also the differentiation between to/cc/bcc,
204 ## so i will keep this.
205 parse_header @source.load_header(@source_info)
206 message_to_chunks @source.load_message(@source_info)
207 rescue SourceError, SocketError, MessageFormatError => e
208 Redwood::log "problem getting messages from #{@source}: #{e.message}"
209 ## we need force_to_top here otherwise this window will cover
210 ## up the error message one
212 Redwood::report_broken_sources :force_to_top => true
213 [Chunk::Text.new(error_message(e.message).split("\n"))]
218 def error_message msg
222 ***********************************************************************
223 An error occurred while loading this message. It is possible that
224 the source has changed, or (in the case of remote sources) is down.
225 You can check the log for errors, though hopefully an error window
226 should have popped up at some point.
228 The message location was:
229 #@source##@source_info
230 ***********************************************************************
232 The error message was:
237 ## wrap any source methods that might throw sourceerrors
238 def with_source_errors_handled
241 rescue SourceError => e
242 Redwood::log "problem getting messages from #{@source}: #{e.message}"
244 Redwood::report_broken_sources :force_to_top => true
245 error_message e.message
250 with_source_errors_handled { @source.raw_header @source_info }
254 with_source_errors_handled { @source.raw_message @source_info }
257 ## much faster than raw_message
258 def each_raw_message_line &b
259 with_source_errors_handled { @source.each_raw_message_line(@source_info, &b) }
262 ## returns all the content from a message that will be indexed
263 def indexable_content
266 from && from.indexable_content,
267 to.map { |p| p.indexable_content },
268 cc.map { |p| p.indexable_content },
269 bcc.map { |p| p.indexable_content },
270 chunks.select { |c| c.is_a? Chunk::Text }.map { |c| c.lines },
271 Message.normalize_subj(subj),
272 ].flatten.compact.join " "
275 def quotable_body_lines
276 chunks.find_all { |c| c.quotable? }.map { |c| c.lines }.flatten
279 def quotable_header_lines
280 ["From: #{@from.full_address}"] +
281 (@to.empty? ? [] : ["To: " + @to.map { |p| p.full_address }.join(", ")]) +
282 (@cc.empty? ? [] : ["Cc: " + @cc.map { |p| p.full_address }.join(", ")]) +
283 (@bcc.empty? ? [] : ["Bcc: " + @bcc.map { |p| p.full_address }.join(", ")]) +
284 ["Date: #{@date.rfc822}",
290 ## here's where we handle decoding mime attachments. unfortunately
291 ## but unsurprisingly, the world of mime attachments is a bit of a
292 ## mess. as an empiricist, i'm basing the following behavior on
293 ## observed mail rather than on interpretations of rfcs, so probably
294 ## this will have to be tweaked.
296 ## the general behavior i want is: ignore content-disposition, at
297 ## least in so far as it suggests something being inline vs being an
298 ## attachment. (because really, that should be the recipient's
299 ## decision to make.) if a mime part is text/plain, OR if the user
300 ## decoding hook converts it, then decode it and display it
301 ## inline. for these decoded attachments, if it has associated
302 ## filename, then make it collapsable and individually saveable;
303 ## otherwise, treat it as regular body text.
305 ## everything else is just an attachment and is not displayed
308 ## so, in contrast to mutt, the user is not exposed to the workings
309 ## of the gruesome slaughterhouse and sausage factory that is a
310 ## mime-encoded message, but need only see the delicious end
313 def multipart_signed_to_chunks m
315 Redwood::log "warning: multipart/signed with #{m.body.size} parts (expecting 2)"
319 payload, signature = m.body
320 if signature.multipart?
321 Redwood::log "warning: multipart/signed with payload multipart #{payload.multipart?} and signature multipart #{signature.multipart?}"
325 ## this probably will never happen
326 if payload.header.content_type == "application/pgp-signature"
327 Redwood::log "warning: multipart/signed with payload content type #{payload.header.content_type}"
331 if signature.header.content_type != "application/pgp-signature"
332 ## unknown signature type; just ignore.
333 #Redwood::log "warning: multipart/signed with signature content type #{signature.header.content_type}"
337 [CryptoManager.verify(payload, signature), message_to_chunks(payload)].flatten.compact
340 def multipart_encrypted_to_chunks m
342 Redwood::log "warning: multipart/encrypted with #{m.body.size} parts (expecting 2)"
346 control, payload = m.body
347 if control.multipart?
348 Redwood::log "warning: multipart/encrypted with control multipart #{control.multipart?} and payload multipart #{payload.multipart?}"
352 if payload.header.content_type != "application/octet-stream"
353 Redwood::log "warning: multipart/encrypted with payload content type #{payload.header.content_type}"
357 if control.header.content_type != "application/pgp-encrypted"
358 Redwood::log "warning: multipart/encrypted with control content type #{signature.header.content_type}"
362 decryptedm, sig, notice = CryptoManager.decrypt payload
363 children = message_to_chunks(decryptedm, true) if decryptedm
364 [notice, sig, children].flatten.compact
367 ## takes a RMail::Message, breaks it into Chunk:: classes.
368 def message_to_chunks m, encrypted=false, sibling_types=[]
371 case m.header.content_type
372 when "multipart/signed"
373 multipart_signed_to_chunks m
374 when "multipart/encrypted"
375 multipart_encrypted_to_chunks m
379 sibling_types = m.body.map { |p| p.header.content_type }
380 chunks = m.body.map { |p| message_to_chunks p, encrypted, sibling_types }.flatten.compact
384 elsif m.header.content_type == "message/rfc822"
385 payload = RMail::Parser.read(m.body)
386 from = payload.header.from.first
387 from_person = from ? Person.from_address(from.format) : nil
388 [Chunk::EnclosedMessage.new(from_person, payload.to_s)] +
389 message_to_chunks(payload, encrypted)
392 ## first, paw through the headers looking for a filename
393 if m.header["Content-Disposition"] && m.header["Content-Disposition"] =~ /filename="?(.*?[^\\])("|;|$)/
395 elsif m.header["Content-Type"] && m.header["Content-Type"] =~ /name="?(.*?[^\\])("|;|$)/
398 ## haven't found one, but it's a non-text message. fake
401 ## TODO: make this less lame.
402 elsif m.header["Content-Type"] && m.header["Content-Type"] !~ /^text\/plain/
404 case m.header["Content-Type"]
405 when /text\/html/: "html"
406 when /image\/(.*)/: $1
409 ["sup-attachment-#{Time.now.to_i}-#{rand 10000}", extension].join(".")
412 ## if there's a filename, we'll treat it as an attachment.
414 # add this to the attachments list if its not a generated html
415 # attachment (should we allow images with generated names?).
416 # Lowercase the filename because searches are easier that way
417 @attachments.push filename.downcase unless filename =~ /^sup-attachment-/
418 add_label :attachment unless filename =~ /^sup-attachment-/
419 content_type = m.header.content_type || "application/unknown" # sometimes RubyMail gives us nil
420 [Chunk::Attachment.new(content_type, filename, m, sibling_types)]
422 ## otherwise, it's body text
424 body = Message.convert_from m.decode, m.charset if m.body
425 text_to_chunks((body || "").normalize_whitespace.split("\n"), encrypted)
430 def self.convert_from body, charset
432 raise MessageFormatError, "RubyMail decode returned a null body" unless body
433 return body unless charset
434 Iconv.easy_decode($encoding, charset, body)
435 rescue Errno::EINVAL, Iconv::InvalidEncoding, Iconv::IllegalSequence, MessageFormatError => e
436 Redwood::log "warning: error (#{e.class.name}) decoding message body from #{charset}: #{e.message}"
437 File.open(File.join(BASE_DIR,"unable-to-decode.txt"), "w") { |f| f.write body }
442 ## parse the lines of text into chunk objects. the heuristics here
443 ## need tweaking in some nice manner. TODO: move these heuristics
444 ## into the classes themselves.
445 def text_to_chunks lines, encrypted
446 state = :text # one of :text, :quote, or :sig
450 lines.each_with_index do |line, i|
451 nextline = lines[(i + 1) ... lines.length].find { |l| l !~ /^\s*$/ } # skip blank lines
457 if line =~ QUOTE_PATTERN || (line =~ QUOTE_START_PATTERN && nextline =~ QUOTE_PATTERN)
459 elsif line =~ SIG_PATTERN && (lines.length - i) < MAX_SIG_DISTANCE
461 elsif line =~ BLOCK_QUOTE_PATTERN
462 newstate = :block_quote
466 chunks << Chunk::Text.new(chunk_lines) unless chunk_lines.empty?
476 if line =~ QUOTE_PATTERN || (line =~ /^\s*$/ && nextline =~ QUOTE_PATTERN)
478 elsif line =~ SIG_PATTERN && (lines.length - i) < MAX_SIG_DISTANCE
485 if chunk_lines.empty?
488 chunks << Chunk::Quote.new(chunk_lines)
494 when :block_quote, :sig
498 if !@have_snippet && state == :text && (@snippet.nil? || @snippet.length < SNIPPET_LEN) && line !~ /[=\*#_-]{3,}/ && line !~ /^\s*$/
500 @snippet += " " unless @snippet.empty?
501 @snippet += line.gsub(/^\s+/, "").gsub(/[\r\n]/, "").gsub(/\s+/, " ")
502 @snippet = @snippet[0 ... SNIPPET_LEN].chomp
503 @dirty = true unless encrypted && $config[:discard_snippets_from_encrypted_messages]
504 @snippet_contains_encrypted_content = true if encrypted
510 when :quote, :block_quote
511 chunks << Chunk::Quote.new(chunk_lines) unless chunk_lines.empty?
513 chunks << Chunk::Text.new(chunk_lines) unless chunk_lines.empty?
515 chunks << Chunk::Signature.new(chunk_lines) unless chunk_lines.empty?