5 ## a Message is what's threaded.
7 ## it is also where the parsing for quotes and signatures is done, but
8 ## that should be moved out to a separate class at some point (because
9 ## i would like, for example, to be able to add in a ruby-talk
10 ## specific module that would detect and link to /ruby-talk:\d+/
11 ## sequences in the text of an email. (how sweet would that be?)
13 ## this class catches all source exceptions. if the underlying source
14 ## throws an error, it is caught and handled.
18 RE_PATTERN = /^((re|re[\[\(]\d[\]\)]):\s*)+/i
20 ## some utility methods
22 def normalize_subj s; s.gsub(RE_PATTERN, ""); end
23 def subj_is_reply? s; s =~ RE_PATTERN; end
24 def reify_subj s; subj_is_reply?(s) ? s : "Re: " + s; end
27 QUOTE_PATTERN = /^\s{0,4}[>|\}]/
28 BLOCK_QUOTE_PATTERN = /^-----\s*Original Message\s*----+$/
29 SIG_PATTERN = /(^-- ?$)|(^\s*----------+\s*$)|(^\s*_________+\s*$)|(^\s*--~--~-)|(^\s*--\+\+\*\*==)/
31 MAX_SIG_DISTANCE = 15 # lines from the end
33 DEFAULT_SENDER = "(missing sender)"
35 attr_reader :id, :date, :from, :subj, :refs, :replytos, :to, :source,
36 :cc, :bcc, :labels, :attachments, :list_address, :recipient_email, :replyto,
37 :source_info, :list_subscribe, :list_unsubscribe
39 bool_reader :dirty, :source_marked_read, :snippet_contains_encrypted_content
41 ## if you specify a :header, will use values from that. otherwise,
42 ## will try and load the header from the source.
44 @source = opts[:source] or raise ArgumentError, "source can't be nil"
45 @source_info = opts[:source_info] or raise ArgumentError, "source_info can't be nil"
46 @snippet = opts[:snippet]
47 @snippet_contains_encrypted_content = false
48 @have_snippet = !(opts[:snippet].nil? || opts[:snippet].empty?)
49 @labels = Set.new(opts[:labels] || [])
55 ## we need to initialize this. see comments in parse_header as to
59 #parse_header(opts[:header] || @source.load_header(@source_info))
62 def parse_header header
63 ## forcibly decode these headers from and to the current encoding,
64 ## which serves to strip out characters that aren't displayable
65 ## (and which would otherwise be screwing up the display)
66 %w(from to subject cc bcc).each do |f|
67 header[f] = Iconv.easy_decode($encoding, $encoding, header[f]) if header[f]
70 @id = if header["message-id"]
71 mid = header["message-id"] =~ /<(.+?)>/ ? $1 : header["message-id"]
72 sanitize_message_id mid
74 id = "sup-faked-" + Digest::MD5.hexdigest(raw_header)
76 #debug "faking non-existent message-id for message from #{from}: #{id}"
80 @from = Person.from_address(if header["from"]
83 name = "Sup Auto-generated Fake Sender <sup@fake.sender.example.com>"
84 #debug "faking non-existent sender for message #@id: #{name}"
88 @date = case(date = header["date"])
94 rescue ArgumentError => e
95 #debug "faking mangled date header for #{@id} (orig #{header['date'].inspect} gave error: #{e.message})"
99 #debug "faking non-existent date header for #{@id}"
103 @subj = header.member?("subject") ? header["subject"].gsub(/\s+/, " ").gsub(/\s+$/, "") : DEFAULT_SUBJECT
104 @to = Person.from_address_list header["to"]
105 @cc = Person.from_address_list header["cc"]
106 @bcc = Person.from_address_list header["bcc"]
108 ## before loading our full header from the source, we can actually
109 ## have some extra refs set by the UI. (this happens when the user
110 ## joins threads manually). so we will merge the current refs values
112 refs = (header["references"] || "").scan(/<(.+?)>/).map { |x| sanitize_message_id x.first }
113 @refs = (@refs + refs).uniq
114 @replytos = (header["in-reply-to"] || "").scan(/<(.+?)>/).map { |x| sanitize_message_id x.first }
116 @replyto = Person.from_address header["reply-to"]
118 if header["list-post"]
119 @list_address = Person.from_address header["list-post"].gsub(/^<mailto:|>$/, "")
124 @recipient_email = header["envelope-to"] || header["x-original-to"] || header["delivered-to"]
125 @source_marked_read = header["status"] == "RO"
126 @list_subscribe = header["list-subscribe"]
127 @list_unsubscribe = header["list-unsubscribe"]
130 ## Expected index entry format:
131 ## :message_id, :subject => String
133 ## :refs, :replytos => Array of String
135 ## :to, :cc, :bcc => Array of Person
136 def load_from_index! entry
137 @id = entry[:message_id]
140 @subj = entry[:subject]
144 @refs = (@refs + entry[:refs]).uniq
145 @replytos = entry[:replytos]
149 @recipient_email = nil
150 @source_marked_read = false
151 @list_subscribe = nil
152 @list_unsubscribe = nil
161 @dirty = true if @refs.delete ref
164 def snippet; @snippet || (chunks && @snippet); end
165 def is_list_message?; !@list_address.nil?; end
166 def is_draft?; @source.is_a? DraftLoader; end
168 raise "not a draft" unless is_draft?
169 @source.fn_for_offset @source_info
172 ## sanitize message ids by removing spaces and non-ascii characters.
173 ## also, truncate to 255 characters. all these steps are necessary
174 ## to make ferret happy. of course, we probably fuck up a couple
175 ## valid message ids as well. as long as we're consistent, this
176 ## should be fine, though.
178 ## also, mostly the message ids that are changed by this belong to
181 ## an alternative would be to SHA1 or MD5 all message ids on a regular basis.
183 def sanitize_message_id mid; mid.gsub(/(\s|[^\000-\177])+/, "")[0..254] end
187 index.update_message_state self
192 def has_label? t; @labels.member? t; end
195 return if @labels.member? l
201 return unless @labels.member? l
211 raise ArgumentError, "not a set" unless l.is_a?(Set)
212 raise ArgumentError, "not a set of labels" unless l.all? { |ll| ll.is_a?(Symbol) }
213 return if @labels == l
223 ## this is called when the message body needs to actually be loaded.
224 def load_from_source!
226 if @source.respond_to?(:has_errors?) && @source.has_errors?
227 [Chunk::Text.new(error_message(@source.error.message).split("\n"))]
230 ## we need to re-read the header because it contains information
231 ## that we don't store in the index. actually i think it's just
232 ## the mailing list address (if any), so this is kinda overkill.
233 ## i could just store that in the index, but i think there might
234 ## be other things like that in the future, and i'd rather not
236 ## actually, it's also the differentiation between to/cc/bcc,
237 ## so i will keep this.
238 parse_header @source.load_header(@source_info)
239 message_to_chunks @source.load_message(@source_info)
240 rescue SourceError, SocketError => e
241 warn "problem getting messages from #{@source}: #{e.message}"
242 ## we need force_to_top here otherwise this window will cover
243 ## up the error message one
245 Redwood::report_broken_sources :force_to_top => true
246 [Chunk::Text.new(error_message(e.message).split("\n"))]
251 def error_message msg
255 ***********************************************************************
256 An error occurred while loading this message. It is possible that
257 the source has changed, or (in the case of remote sources) is down.
258 You can check the log for errors, though hopefully an error window
259 should have popped up at some point.
261 The message location was:
262 #@source##@source_info
263 ***********************************************************************
265 The error message was:
270 ## wrap any source methods that might throw sourceerrors
271 def with_source_errors_handled
274 rescue SourceError => e
275 warn "problem getting messages from #{@source}: #{e.message}"
277 Redwood::report_broken_sources :force_to_top => true
278 error_message e.message
283 with_source_errors_handled { @source.raw_header @source_info }
287 with_source_errors_handled { @source.raw_message @source_info }
290 ## much faster than raw_message
291 def each_raw_message_line &b
292 with_source_errors_handled { @source.each_raw_message_line(@source_info, &b) }
295 ## returns all the content from a message that will be indexed
296 def indexable_content
299 from && from.indexable_content,
300 to.map { |p| p.indexable_content },
301 cc.map { |p| p.indexable_content },
302 bcc.map { |p| p.indexable_content },
303 indexable_chunks.map { |c| c.lines },
305 ].flatten.compact.join " "
309 indexable_chunks.map { |c| c.lines }.flatten.compact.join " "
313 chunks.select { |c| c.is_a? Chunk::Text }
316 def indexable_subject
317 Message.normalize_subj(subj)
320 def quotable_body_lines
321 chunks.find_all { |c| c.quotable? }.map { |c| c.lines }.flatten
324 def quotable_header_lines
325 ["From: #{@from.full_address}"] +
326 (@to.empty? ? [] : ["To: " + @to.map { |p| p.full_address }.join(", ")]) +
327 (@cc.empty? ? [] : ["Cc: " + @cc.map { |p| p.full_address }.join(", ")]) +
328 (@bcc.empty? ? [] : ["Bcc: " + @bcc.map { |p| p.full_address }.join(", ")]) +
329 ["Date: #{@date.rfc822}",
333 def self.build_from_source source, source_info
334 m = Message.new :source => source, :source_info => source_info
341 ## here's where we handle decoding mime attachments. unfortunately
342 ## but unsurprisingly, the world of mime attachments is a bit of a
343 ## mess. as an empiricist, i'm basing the following behavior on
344 ## observed mail rather than on interpretations of rfcs, so probably
345 ## this will have to be tweaked.
347 ## the general behavior i want is: ignore content-disposition, at
348 ## least in so far as it suggests something being inline vs being an
349 ## attachment. (because really, that should be the recipient's
350 ## decision to make.) if a mime part is text/plain, OR if the user
351 ## decoding hook converts it, then decode it and display it
352 ## inline. for these decoded attachments, if it has associated
353 ## filename, then make it collapsable and individually saveable;
354 ## otherwise, treat it as regular body text.
356 ## everything else is just an attachment and is not displayed
359 ## so, in contrast to mutt, the user is not exposed to the workings
360 ## of the gruesome slaughterhouse and sausage factory that is a
361 ## mime-encoded message, but need only see the delicious end
364 def multipart_signed_to_chunks m
366 warn "multipart/signed with #{m.body.size} parts (expecting 2)"
370 payload, signature = m.body
371 if signature.multipart?
372 warn "multipart/signed with payload multipart #{payload.multipart?} and signature multipart #{signature.multipart?}"
376 ## this probably will never happen
377 if payload.header.content_type == "application/pgp-signature"
378 warn "multipart/signed with payload content type #{payload.header.content_type}"
382 if signature.header.content_type != "application/pgp-signature"
383 ## unknown signature type; just ignore.
384 #warn "multipart/signed with signature content type #{signature.header.content_type}"
388 [CryptoManager.verify(payload, signature), message_to_chunks(payload)].flatten.compact
391 def multipart_encrypted_to_chunks m
393 warn "multipart/encrypted with #{m.body.size} parts (expecting 2)"
397 control, payload = m.body
398 if control.multipart?
399 warn "multipart/encrypted with control multipart #{control.multipart?} and payload multipart #{payload.multipart?}"
403 if payload.header.content_type != "application/octet-stream"
404 warn "multipart/encrypted with payload content type #{payload.header.content_type}"
408 if control.header.content_type != "application/pgp-encrypted"
409 warn "multipart/encrypted with control content type #{signature.header.content_type}"
413 notice, sig, decryptedm = CryptoManager.decrypt payload
414 if decryptedm # managed to decrypt
415 children = message_to_chunks(decryptedm, true)
416 [notice, sig, children]
422 ## takes a RMail::Message, breaks it into Chunk:: classes.
423 def message_to_chunks m, encrypted=false, sibling_types=[]
426 case m.header.content_type
427 when "multipart/signed"
428 multipart_signed_to_chunks m
429 when "multipart/encrypted"
430 multipart_encrypted_to_chunks m
434 sibling_types = m.body.map { |p| p.header.content_type }
435 chunks = m.body.map { |p| message_to_chunks p, encrypted, sibling_types }.flatten.compact
439 elsif m.header.content_type == "message/rfc822"
441 payload = RMail::Parser.read(m.body)
442 from = payload.header.from.first ? payload.header.from.first.format : ""
443 to = payload.header.to.map { |p| p.format }.join(", ")
444 cc = payload.header.cc.map { |p| p.format }.join(", ")
445 subj = payload.header.subject
446 subj = subj ? Message.normalize_subj(payload.header.subject.gsub(/\s+/, " ").gsub(/\s+$/, "")) : subj
447 if Rfc2047.is_encoded? subj
448 subj = Rfc2047.decode_to $encoding, subj
450 msgdate = payload.header.date
451 from_person = from ? Person.from_address(from) : nil
452 to_people = to ? Person.from_address_list(to) : nil
453 cc_people = cc ? Person.from_address_list(cc) : nil
454 [Chunk::EnclosedMessage.new(from_person, to_people, cc_people, msgdate, subj)] + message_to_chunks(payload, encrypted)
456 [Chunk::EnclosedMessage.new(nil, "")]
460 ## first, paw through the headers looking for a filename
461 if m.header["Content-Disposition"] && m.header["Content-Disposition"] =~ /filename="?(.*?[^\\])("|;|$)/
463 elsif m.header["Content-Type"] && m.header["Content-Type"] =~ /name="?(.*?[^\\])("|;|$)/
466 ## haven't found one, but it's a non-text message. fake
469 ## TODO: make this less lame.
470 elsif m.header["Content-Type"] && m.header["Content-Type"] !~ /^text\/plain/
472 case m.header["Content-Type"]
473 when /text\/html/ then "html"
474 when /image\/(.*)/ then $1
477 ["sup-attachment-#{Time.now.to_i}-#{rand 10000}", extension].join(".")
480 ## if there's a filename, we'll treat it as an attachment.
482 # add this to the attachments list if its not a generated html
483 # attachment (should we allow images with generated names?).
484 # Lowercase the filename because searches are easier that way
485 @attachments.push filename.downcase unless filename =~ /^sup-attachment-/
486 add_label :attachment unless filename =~ /^sup-attachment-/
487 content_type = m.header.content_type || "application/unknown" # sometimes RubyMail gives us nil
488 [Chunk::Attachment.new(content_type, filename, m, sibling_types)]
490 ## otherwise, it's body text
492 ## if there's no charset, use the current encoding as the charset.
493 ## this ensures that the body is normalized to avoid non-displayable
495 body = Iconv.easy_decode($encoding, m.charset || $encoding, m.decode) if m.body
496 text_to_chunks((body || "").normalize_whitespace.split("\n"), encrypted)
501 ## parse the lines of text into chunk objects. the heuristics here
502 ## need tweaking in some nice manner. TODO: move these heuristics
503 ## into the classes themselves.
504 def text_to_chunks lines, encrypted
505 state = :text # one of :text, :quote, or :sig
509 lines.each_with_index do |line, i|
510 nextline = lines[(i + 1) ... lines.length].find { |l| l !~ /^\s*$/ } # skip blank lines
516 ## the following /:$/ followed by /\w/ is an attempt to detect the
517 ## start of a quote. this is split into two regexen because the
518 ## original regex /\w.*:$/ had very poor behavior on long lines
519 ## like ":a:a:a:a:a" that occurred in certain emails.
520 if line =~ QUOTE_PATTERN || (line =~ /:$/ && line =~ /\w/ && nextline =~ QUOTE_PATTERN)
522 elsif line =~ SIG_PATTERN && (lines.length - i) < MAX_SIG_DISTANCE
524 elsif line =~ BLOCK_QUOTE_PATTERN
525 newstate = :block_quote
529 chunks << Chunk::Text.new(chunk_lines) unless chunk_lines.empty?
539 if line =~ QUOTE_PATTERN || (line =~ /^\s*$/ && nextline =~ QUOTE_PATTERN)
541 elsif line =~ SIG_PATTERN && (lines.length - i) < MAX_SIG_DISTANCE
548 if chunk_lines.empty?
551 chunks << Chunk::Quote.new(chunk_lines)
557 when :block_quote, :sig
561 if !@have_snippet && state == :text && (@snippet.nil? || @snippet.length < SNIPPET_LEN) && line !~ /[=\*#_-]{3,}/ && line !~ /^\s*$/
563 @snippet += " " unless @snippet.empty?
564 @snippet += line.gsub(/^\s+/, "").gsub(/[\r\n]/, "").gsub(/\s+/, " ")
565 @snippet = @snippet[0 ... SNIPPET_LEN].chomp
566 @dirty = true unless encrypted && $config[:discard_snippets_from_encrypted_messages]
567 @snippet_contains_encrypted_content = true if encrypted
573 when :quote, :block_quote
574 chunks << Chunk::Quote.new(chunk_lines) unless chunk_lines.empty?
576 chunks << Chunk::Text.new(chunk_lines) unless chunk_lines.empty?
578 chunks << Chunk::Signature.new(chunk_lines) unless chunk_lines.empty?