encoding and rfc2047 header support

git-svn-id: svn://rubyforge.org/var/svn/sup/trunk@463 5c8cc53c-5e98-4d25-b20a-d8db53a31250

diff --git a/Manifest.txt b/Manifest.txt
index 8d90c399bee698055be6a67417e9f27824c6853d..440f0bba8cd8501a995593d0ceb3cb59883b09df 100644 (file)
--- a/Manifest.txt
+++ b/Manifest.txt
@@ -55,6 +55,7 @@ lib/sup/modes/thread-index-mode.rb
 lib/sup/modes/thread-view-mode.rb
 lib/sup/person.rb
 lib/sup/poll.rb
+lib/sup/rfc2047.rb
 lib/sup/sent.rb
 lib/sup/source.rb
 lib/sup/suicide.rb

diff --git a/Rakefile b/Rakefile
index cc4a12a1b844a490ac8bc14f09cb85cdb91edfdd..0ea68f35807f3dac88a99fccf31af96fc64a017c 100644 (file)
--- a/Rakefile
+++ b/Rakefile
@@ -16,7 +16,7 @@ Hoe.new('sup', Redwood::VERSION) do |p|
   p.url = p.paragraphs_of('README.txt', 0).first.split(/\n/)[2].gsub(/^\s+/, "")
   p.changes = p.paragraphs_of('History.txt', 0..0).join("\n\n")
   p.email = "wmorgan-sup@masanjin.net"
-  p.extra_deps = [['ferret', '>= 0.10.13'], ['ncurses', '>= 0.9.1'], ['rmail', '>= 0.17'], 'highline', 'net-ssh', ['trollop', '>= 1.5'], 'lockfile']
+  p.extra_deps = [['ferret', '>= 0.10.13'], ['ncurses', '>= 0.9.1'], ['rmail', '>= 0.17'], 'highline', 'net-ssh', ['trollop', '>= 1.5'], 'lockfile', 'iconv']
 end
 
 rule 'ss?.png' => 'ss?-small.png' do |t|

diff --git a/lib/sup.rb b/lib/sup.rb
index b7c8f0212c6d18713abfe65502e5b80d20798f20..3f305ce29a7e3bcf3d4f7e108dfd7cc0bfd8955d 100644 (file)
--- a/lib/sup.rb
+++ b/lib/sup.rb
@@ -47,6 +47,16 @@ module Redwood
   YAML_DOMAIN = "masanjin.net"
   YAML_DATE = "2006-10-01"
 
+## determine encoding and character set
+## probably a better way to do this
+  $ctype = ENV["LC_CTYPE"] || ENV["LANG"] || "en-US.utf-8"
+  $encoding =
+    if $ctype =~ /\.(.*)?/
+      $1
+    else
+      "utf-8"
+    end
+
 ## record exceptions thrown in threads nicely
   $exception = nil
   def reporting_thread

diff --git a/lib/sup/mbox.rb b/lib/sup/mbox.rb
index 894d56303ea959ce707754c5a4bd0492d0f8198b..97ddcbb84e03e4815038ee6b3c96f77b1dadf91c 100644 (file)
--- a/lib/sup/mbox.rb
+++ b/lib/sup/mbox.rb
@@ -1,10 +1,14 @@
 require "sup/mbox/loader"
 require "sup/mbox/ssh-file"
 require "sup/mbox/ssh-loader"
+require "sup/rfc2047"
 
 module Redwood
 
-## some utility functions
+## some utility functions. actually these are not mbox-specific at all
+## and should be moved somewhere else.
+##
+## TODO: move functionality to somewhere better, like message.rb
 module MBox
   BREAK_RE = /^From \S+/
 
@@ -46,6 +50,16 @@ module MBox
       header[mid_field] = $1
     end
 
+    header.each do |k, v|
+      next unless Rfc2047.is_encoded? v
+      header[k] =
+        begin
+          Rfc2047.decode_to $encoding, v
+        rescue Errno::EINVAL, Icon::InvalidEncoding, Iconv::IllegalSequence => e
+          Redwood::log "warning: error decoding RFC 2047 header: #{e.message}"
+          v
+        end
+    end
     header
   end
   

diff --git a/lib/sup/message.rb b/lib/sup/message.rb
index 711ab2e5fc4a4fbcda977c9510f6699580d6b514..f1041274952f52e6f09dd18f9e7033add729fabf 100644 (file)
--- a/lib/sup/message.rb
+++ b/lib/sup/message.rb
@@ -1,5 +1,6 @@
 require 'tempfile'
 require 'time'
+require 'iconv'
 
 module Redwood
 
@@ -268,7 +269,21 @@ private
     else
       case m.header.content_type
       when "text/plain", nil
+        charset = 
+          if m.header.field?("content-type") && m.header.fetch("content-type") =~ /charset=(.*?)(;|$)/
+            $1
+          end
+
         m.body && body = m.decode or raise MessageFormatError, "For some bizarre reason, RubyMail was unable to parse this message."
+
+        if charset
+          begin
+            body = Iconv.iconv($encoding, charset, body).join
+          rescue Errno::EINVAL, Icon::InvalidEncoding, Iconv::IllegalSequence => e
+            Redwood::log "warning: error decoding message body from #{charset}: #{e.message}"
+          end
+        end
+
         text_to_chunks(body.normalize_whitespace.split("\n"))
       when /^multipart\//
         []

diff --git a/lib/sup/rfc2047.rb b/lib/sup/rfc2047.rb
new file mode 100644 (file)
index 0000000..ab006a0
--- /dev/null
+++ b/lib/sup/rfc2047.rb
@@ -0,0 +1,61 @@
+## from: http://blade.nagaokaut.ac.jp/cgi-bin/scat.rb/ruby/ruby-talk/101949
+
+# $Id: rfc2047.rb,v 1.4 2003/04/18 20:55:56 sam Exp $
+# MODIFIED slightly by William Morgan
+#
+# An implementation of RFC 2047 decoding.
+#
+# This module depends on the iconv library by Nobuyoshi Nakada, which I've 
+# heard may be distributed as a standard part of Ruby 1.8. Many thanks to him
+# for helping with building and using iconv.
+#
+# Thanks to "Josef 'Jupp' Schugt" <jupp / gmx.de> for pointing out an error with
+# stateful character sets.
+#
+# Copyright (c) Sam Roberts <sroberts / uniserve.com> 2004
+#
+# This file is distributed under the same terms as Ruby.
+
+require 'iconv'
+
+module Rfc2047
+  WORD = %r{=\?([!\#$%&'*+-/0-9A-Z\\^\`a-z{|}~]+)\?([BbQq])\?([!->@-~]+)\?=} # :nodoc: 'stupid ruby-mode
+  WORDSEQ = %r{(#{WORD.source})\s+(?=#{WORD.source})}
+
+  def Rfc2047.is_encoded? s; s =~ WORD end
+
+  # Decodes a string, +from+, containing RFC 2047 encoded words into a target
+  # character set, +target+. See iconv_open(3) for information on the
+  # supported target encodings. If one of the encoded words cannot be
+  # converted to the target encoding, it is left in its encoded form.
+  def Rfc2047.decode_to(target, from)
+    from = from.gsub(WORDSEQ, '\1')
+    out = from.gsub(WORD) do
+      |word|
+      charset, encoding, text = $1, $2, $3
+
+      # B64 or QP decode, as necessary:
+      case encoding
+        when 'b', 'B'
+          #puts text
+          text = text.unpack('m*')[0]
+          #puts text.dump
+
+        when 'q', 'Q'
+          # RFC 2047 has a variant of quoted printable where a ' ' character
+          # can be represented as an '_', rather than =32, so convert
+          # any of these that we find before doing the QP decoding.
+          text = text.tr("_", " ")
+          text = text.unpack('M*')[0]
+
+        # Don't need an else, because no other values can be matched in a
+        # WORD.
+      end
+
+      # Convert:
+      #
+      # Remember - Iconv.open(to, from)!
+      text = Iconv.iconv(target, charset, text).join
+    end
+  end
+end