From bd0c43c067eca78ea602dbaf018e3a284e18cd4c Mon Sep 17 00:00:00 2001
From: wmorgan <wmorgan@5c8cc53c-5e98-4d25-b20a-d8db53a31250>
Date: Thu, 28 Dec 2006 21:36:00 +0000
Subject: [PATCH] added mbox+ssh support (though pretty lame at the moment)
 also changed sup-impor to use highline

git-svn-id: svn://rubyforge.org/var/svn/sup/trunk@108 5c8cc53c-5e98-4d25-b20a-d8db53a31250
---
 Manifest.txt               |   2 +
 Rakefile                   |   2 +-
 bin/sup-import             |  18 ++-
 lib/sup/mbox.rb            |   2 +
 lib/sup/mbox/ssh-file.rb   | 222 +++++++++++++++++++++++++++++++++++++
 lib/sup/mbox/ssh-loader.rb |  37 +++++++
 6 files changed, 277 insertions(+), 6 deletions(-)
 create mode 100644 lib/sup/mbox/ssh-file.rb
 create mode 100644 lib/sup/mbox/ssh-loader.rb

diff --git a/Manifest.txt b/Manifest.txt
index 0fd2577..f69951d 100644
--- a/Manifest.txt
+++ b/Manifest.txt
@@ -9,6 +9,8 @@ doc/TODO
 bin/sup
 bin/sup-import
 lib/sup/mbox/loader.rb
+lib/sup/mbox/ssh-file.rb
+lib/sup/mbox/ssh-loader.rb
 lib/sup/modes/line-cursor-mode.rb
 lib/sup/modes/reply-mode.rb
 lib/sup/modes/scroll-mode.rb
diff --git a/Rakefile b/Rakefile
index c9bceea..1eb3518 100644
--- a/Rakefile
+++ b/Rakefile
@@ -12,7 +12,7 @@ Hoe.new('sup', Redwood::VERSION) do |p|
   p.url = p.paragraphs_of('README.txt', 0).first.split(/\n/)[2].gsub(/^\s+/, "")
   p.changes = p.paragraphs_of('History.txt', 0..1).join("\n\n")
   p.email = "wmorgan-sup@masanjin.net"
-  p.extra_deps = [['ferret', '>= 0.10.13'], ['ncurses', '>= 0.9.1'], ['rmail', '>= 0.17']]
+  p.extra_deps = [['ferret', '>= 0.10.13'], ['ncurses', '>= 0.9.1'], ['rmail', '>= 0.17'], 'highline']
 end
 
 rule 'ss?.png' => 'ss?-small.png' do |t|
diff --git a/bin/sup-import b/bin/sup-import
index 798e12c..628367a 100644
--- a/bin/sup-import
+++ b/bin/sup-import
@@ -1,5 +1,7 @@
 #!/usr/bin/env ruby
 
+require 'rubygems'
+require 'highline'
 require "sup"
 
 class Float
@@ -84,17 +86,23 @@ index = Redwood::Index.new
 index.load
 puts "loaded index of #{index.size} messages"
 
+h = HighLine.new
+
 sources = ARGV.map do |fn|
   fn = "mbox://#{fn}" unless fn =~ %r!://!
   source = index.source_for fn
   unless source
     source = 
       case fn
+      when %r!^mbox\+ssh://!
+        username = h.ask("Username for #{fn}: ");
+        password = h.ask("Password for #{fn}: ") { |q| q.echo = false }
+        puts # why?
+        Redwood::MBox::SSHLoader.new(fn, username, password, nil, !unusual, !!archive)
       when %r!^imaps?://!
-        print "Username for #{fn}: "
-        username = $stdin.gets.chomp
-        print "Password for #{fn} (warning: cleartext): "
-        password = $stdin.gets.chomp
+        username = h.ask("Username for #{fn}: ");
+        password = h.ask("Password for #{fn}: ") { |q| q.echo = false }
+        puts # why?
         Redwood::IMAP.new(fn, username, password, nil, !unusual, !!archive)
       else
         Redwood::MBox::Loader.new(fn, nil, !unusual, !!archive)
@@ -138,7 +146,7 @@ begin
           found[m.id] = true
         end
         m.remove_label :unread if m.status == "RO" unless force_read
-        #puts "# message at #{offset}, labels: #{labels * ', '}" unless rebuild || force_rebuild
+        puts "# message at #{offset}, labels: #{labels * ', '}" unless rebuild || force_rebuild
         if (rebuild || force_rebuild) && 
             (docid, entry = index.load_entry_for_id(m.id)) && entry
           if force_rebuild || entry[:source_info].to_i != offset
diff --git a/lib/sup/mbox.rb b/lib/sup/mbox.rb
index 4d53e78..04cd2f8 100644
--- a/lib/sup/mbox.rb
+++ b/lib/sup/mbox.rb
@@ -1,4 +1,6 @@
 require "sup/mbox/loader"
+require "sup/mbox/ssh-file"
+require "sup/mbox/ssh-loader"
 
 module Redwood
 
diff --git a/lib/sup/mbox/ssh-file.rb b/lib/sup/mbox/ssh-file.rb
new file mode 100644
index 0000000..08a4013
--- /dev/null
+++ b/lib/sup/mbox/ssh-file.rb
@@ -0,0 +1,222 @@
+require 'net/ssh'
+
+module Redwood
+module MBox
+
+class SSHFileError < StandardError; end
+
+## this is a file-like interface to a file that actually lives on the
+## other end of an ssh connection. it works by using wc, head and tail
+## to simulate (buffered) random access.
+
+## it doesn't work very well, because while on a fast connection ssh
+## can have a nice bandwidth, the latency is pretty terrible. and
+## since reading mbox files involves jumping around a lot (they are
+## very verbose), it is tragically slow to do anything. i've tried to
+## compensate by caching massive amounts of data, but that doesn't
+## really help. your best bet for remote file access remains IMAP.
+## i'm going to include this in the codebase for the time begin,
+## because maybe someone very motivated can put some energy into a
+## better approach (probably one that doesn't involve the synchronous
+## shell.)
+
+## there are two kinds of file access that are typical in sup: the
+## first is an import, which starts at some point in the file and
+## reads until the end. the other is during loading time, which does
+## arbitrary reads into the file, but typically reads *backwards* in
+## the file (because messages are loaded and displayed most recent
+## first, and typically later message are later in the mbox file).  so
+## we have to be careful that whatever caching we do supports both.
+
+$f = File.open("asdf.txt", "w")
+def debuggg s
+  $f.puts s
+  $f.flush
+end
+module_function :debuggg
+
+
+class Buffer
+  def initialize
+    clear!
+  end
+
+  def clear!
+    MBox::debuggg ">>> CLEARING <<<"
+    @start = nil
+    @buf = ""
+  end
+
+  def empty?; @start.nil?; end
+  def start; @start; end
+  def endd; @start + @buf.length; end
+
+  def add data, offset=endd
+    MBox::debuggg "+ adding #{data.length} bytes; size will be #{size + data.length}; limit #{SSHFile::MAX_BUF_SIZE}"
+
+    if start.nil?
+      @buf = data
+      @start = offset
+      return
+    end
+
+    raise "non-continguous data added to buffer (data #{offset}:#{offset + data.length}, buf range #{start}:#{endd})" if offset + data.length < start || offset > endd
+
+    if offset < start
+      @buf = data[0 ... (start - offset)] + @buf
+      @start = offset
+    else
+      return if offset + data.length < endd
+      @buf += data[(endd - offset) .. -1]
+    end
+  end
+
+  def [](o)
+    raise "only ranges supported due to programmer's laziness" unless o.is_a? Range
+    @buf[Range.new(o.first - @start, o.last - @start, o.exclude_end?)]
+  end
+
+  def index what, start=0
+    x = @buf.index(what, start - @start)
+    x.nil? ? nil : x + @start
+  end
+  def rindex what, start=0
+    x = @buf.rindex(what, start - @start)
+    x.nil? ? nil : x + @start
+  end
+
+  def size; empty? ? 0 : @buf.size; end
+  def to_s; empty? ? "<empty>" : "[#{start}, #{endd})"; end # for debugging
+end
+
+class SSHFile
+  MAX_BUF_SIZE = 1024 * 1024 * 3
+  MAX_TRANSFER_SIZE = 1024 * 256 # bytes
+  REASONABLE_TRANSFER_SIZE = 1024 * 128 # bytes
+  SIZE_CHECK_INTERVAL = 60 * 1 # seconds
+
+  def initialize host, fn, ssh_opts={}
+    @buf = Buffer.new
+    @host = host
+    @fn = fn
+    @ssh_opts = ssh_opts
+    @file_size = nil
+  end
+
+  def connect
+    return if @session
+    # MBox::debuggg "starting session..."
+    @session = Net::SSH.start @host, @ssh_opts
+    # MBox::debuggg "starting shell..."
+    @shell = @session.shell.sync
+    # MBox::debuggg "ready for heck!"
+    raise Errno::ENOENT, @fn unless @shell.test("-e #@fn").status == 0
+  end
+
+
+  def eof?; @offset >= size; end
+  def eof; eof?; end # lame but IO does this and rmail depends on it
+
+  def seek loc
+    # MBox::debuggg "seeking to #{loc} from #@offset"
+    @offset = loc
+  end
+  def tell; @offset; end
+  def total; size; end
+
+  def size
+    if @file_size.nil? || (Time.now - @last_size_check) > SIZE_CHECK_INTERVAL
+      @last_size_check = Time.now
+      @file_size = do_remote("wc -c #@fn").split.first.to_i
+    end
+    @file_size
+  end
+
+  def gets
+    return nil if eof?
+
+    make_buf_include @offset
+    expand_buf_forward while @buf.index("\n", @offset).nil? && @buf.endd < size
+
+    line = @buf[@offset .. (@buf.index("\n", @offset) || -1)]
+    @offset += line.length
+    # MBox::debuggg "gets is of length #{line.length}, offset now #@offset"
+    line
+  end
+
+  def read n
+    return nil if eof?
+
+    make_buf_include @offset, n
+    @buf[@offset ... (@offset += n)]
+  end
+
+private
+
+  def do_remote cmd
+    retries = 0
+    connect
+    MBox::debuggg "sending command: #{cmd.inspect}"
+    begin
+      result = @shell.send_command cmd
+      raise SSHFileError, "Unable to perform remote command #{cmd.inspect}: #{result.stderr[0 .. 100]}" unless result.status == 0
+    rescue Net::SSH::Exception
+      retry if (retries += 1) < 3
+      raise
+    end
+    result.stdout
+  end
+
+  def get_bytes offset, size
+    MBox::debuggg "get_bytes(#{offset}, #{size})"
+    MBox::debuggg "! request for [#{offset}, #{offset + size}); buf is #@buf"
+    raise "wtf: offset #{offset} size #{size}" if size == 0 || offset < 0
+    do_remote("tail -c +#{offset + 1} #@fn | head -c #{size}")
+  end
+
+  def expand_buf_forward n=REASONABLE_TRANSFER_SIZE
+    @buf.add get_bytes(@buf.endd, n)
+    # trim if necessary
+  end
+
+  ## try our best to transfer somewhere between
+  ## REASONABLE_TRANSFER_SIZE and MAX_TRANSFER_SIZE bytes
+  def make_buf_include offset, size=0
+    good_size = [size, REASONABLE_TRANSFER_SIZE].max
+    remainder = good_size - size
+
+    trans_start, trans_size = 
+      if @buf.empty?
+        [[offset - (remainder / 2), 0].max, good_size]
+      elsif offset < @buf.start
+        if @buf.start - offset <= good_size
+          start = [@buf.start - good_size, 0].max
+          [start, @buf.start - start]
+        elsif @buf.start - offset < MAX_TRANSFER_SIZE
+          [offset, @buf.start - offset]
+        else
+          MBox::debuggg "clearing buffer because buf.start #{@buf.start} - offset #{offset} >= #{MAX_TRANSFER_SIZE}"
+          @buf.clear!
+          [[offset - (remainder / 2), 0].max, good_size]
+        end
+      else
+        return if [offset + size, self.size].min <= @buf.endd # whoohoo!
+        if offset - @buf.endd <= good_size
+          [@buf.endd, good_size]
+        elsif offset - @buf.endd < MAX_TRANSFER_SIZE
+          [@buf.endd, offset - @buf.endd]
+        else
+          MBox::debuggg "clearing buffer because offset #{offset} - buf.end #{@buf.endd} >= #{MAX_TRANSFER_SIZE}"
+          @buf.clear!
+          [[offset - (remainder / 2), 0].max, good_size]
+        end
+      end          
+
+    MBox::debuggg "make_buf_include(#{offset}, #{size})"
+    @buf.clear! if @buf.size > MAX_BUF_SIZE
+    @buf.add get_bytes(trans_start, trans_size), trans_start
+  end
+end
+
+end
+end
diff --git a/lib/sup/mbox/ssh-loader.rb b/lib/sup/mbox/ssh-loader.rb
new file mode 100644
index 0000000..6a41900
--- /dev/null
+++ b/lib/sup/mbox/ssh-loader.rb
@@ -0,0 +1,37 @@
+require 'net/ssh'
+
+module Redwood
+module MBox
+
+class SSHLoader < Loader
+  def initialize uri, username=nil, password=nil, start_offset=nil, usual=true, archived=false, id=nil
+    raise ArgumentError, "not an mbox+ssh uri" unless uri =~ %r!^mbox\+ssh://!
+
+    @parsed_uri = URI(uri)
+    @username = username
+    @password = password
+    @f = nil
+
+    opts = {}
+    opts[:username] = @username if @username
+    opts[:password] = @password if @password
+    
+    @f = SSHFile.new host, filename, opts
+    super @f, start_offset, usual, archived, id
+    @uri = uri
+    ## heuristic: use the filename as a label, unless the file
+    ## has a path that probably represents an inbox.
+    @labels << File.basename(filename).intern unless File.dirname(filename) =~ /\b(var|usr|spool)\b/
+  end
+
+  def host; @parsed_uri.host; end
+  def filename; @parsed_uri.path[1..-1] end ##XXXX TODO handle nil
+
+  def end_offset; @f.size; end
+  def to_s; @parsed_uri.to_s; end
+end
+
+Redwood::register_yaml(SSHLoader, %w(uri username password cur_offset usual archived id))
+
+end
+end
-- 
2.43.0