bm.draw_screen
imode.load_more_threads ibuf.content_height
- ::Thread.new { sleep 5; PollManager.poll }
+ ::Thread.new { sleep 3; PollManager.poll }
until $exception
bm.draw_screen
raise $exception
end
-
end
-
puts "loading index..."
index = Redwood::Index.new
index.load
-pre_nm = index.size
puts "loaded index of #{index.size} messages"
sources = ARGV.map do |fn|
labels -= [:inbox] if force_archive
labels -= [:unread] if force_read
begin
- m = Redwood::Message.new source, offset, labels
+ m = Redwood::Message.new :source => source, :source_info => offset, :labels => labels
if found[m.id]
puts "skipping duplicate message #{m.id}"
next
else
num += 1 if index.add_message m
end
- rescue Redwood::MessageFormatError => e
+ rescue Redwood::MessageFormatError, Redwood::MBox::Error => e
$stderr.puts "ignoring erroneous message at #{source}##{offset}: #{e.message}"
end
if num % 1000 == 0 && num > 0
--- /dev/null
+#!/usr/bin/env ruby
+
+require 'optparse'
+
+$opts = {
+ :unusual => false,
+ :archive => false,
+ :scan_num => 10,
+}
+
+
+OPTIONPARSERSUCKS = "\n" + " " * 38
+OptionParser.new do |opts|
+ opts.banner = <<EOS
+Usage: sup-recover-sources [options] <source>+
+
+Rebuilds a lost sources.yaml file by reading messages from a list of
+sources and determining, for each source, the most prevalent
+'source_id' field of messages from that source in the index.
+
+The only non-deterministic component to this is that if the same
+message appears in multiple sources, those sources may be
+mis-diagnosed by this program.
+
+If the first N messages (--scan-num below) all have the same source_id
+in the index, the source will be added to sources.yaml. Otherwise, the
+distribution will be printed, and you will have to add it by hand.
+
+The offset pointer into the sources will be set to the end of the source,
+so you will have to run sup-import --rebuild for each new source after
+doing this.
+
+Options include:
+EOS
+
+ opts.on("--unusual", "Mark sources as 'unusual'. Only usual#{OPTIONPARSERSUCKS}sources will be polled by hand. Default:#{OPTIONPARSERSUCKS}#{$opts[:unusual]}.") { $opts[:unusual] = true }
+
+ opts.on("--archive", "Mark sources as 'archive'. New messages#{OPTIONPARSERSUCKS}from these sources will not appear in#{OPTIONPARSERSUCKS}the inbox. Default: #{$opts[:archive]}.") { $opts[:archive] = true }
+
+ opts.on("--scan-num N", Integer, "Number of messages to scan per source.#{OPTIONPARSERSUCKS}Default: #{$opts[:scan_num]}.") do |n|
+ $opts[:scan_num] = n
+ end
+
+ opts.on_tail("-h", "--help", "Show this message") do
+ puts opts
+ exit
+ end
+end.parse(ARGV)
+
+require "sup"
+puts "loading index..."
+index = Redwood::Index.new
+index.load
+puts "loaded index of #{index.size} messages"
+
+ARGV.each do |fn|
+ next if index.source_for fn
+
+ ## TODO: merge this code with the same snippet in import
+ source =
+ case fn
+ when %r!^imaps?://!
+ print "Username for #{fn}: "
+ username = $stdin.gets.chomp
+ print "Password for #{fn} (warning: cleartext): "
+ password = $stdin.gets.chomp
+ Redwood::IMAP.new(fn, username, password, nil, !$opts[:unusual], $opts[:archive])
+ else
+ Redwood::MBox::Loader.new(fn, nil, !$opts[:unusual], $opts[:archive])
+ end
+
+ source_ids = {}
+ count = 0
+ source.each do |offset, labels|
+ begin
+ m = Redwood::Message.new :source => source, :source_info => offset
+ docid, entry = index.load_entry_for_id m.id
+ next unless entry
+ #puts "# #{source} #{offset} #{entry[:source_id]}"
+
+ source_ids[entry[:source_id]] = (source_ids[entry[:source_id]] || 0) + 1
+ count += 1
+ break if count == $opts[:scan_num]
+ rescue Redwood::MessageFormatError => e
+ puts "# #{e.message}"
+ end
+ end
+
+ if source_ids.size == 1
+ id = source_ids.keys.first.to_i
+ puts "assigned #{source} to #{source_ids.keys.first}"
+ source.id = id
+ source.seek_to! source.total
+ index.add_source source
+ else
+ puts ">> unable to determine #{source}: #{source_ids.inspect}"
+ end
+end
+
+index.save
YAML_DATE = "2006-10-01"
## one-stop shop for yamliciousness
-
def register_yaml klass, props
vars = props.map { |p| "@#{p}" }
path = klass.name.gsub(/::/, "/")
end
## set up default configuration file
-
if File.exists? Redwood::CONFIG_FN
$config = Redwood::load_yaml_obj Redwood::CONFIG_FN
else
require "sup/update"
require "sup/message"
require "sup/mbox"
+require "sup/imap"
require "sup/person"
require "sup/account"
require "sup/thread"
File.open(fn, "w") { |f| yield f }
@source.each do |offset, labels|
- m = Message.new @source, offset, labels
+ m = Message.new :source => @source, :source_info => offset, :labels => labels
Index.add_message m
UpdateManager.relay :add, m
end
--- /dev/null
+require 'uri'
+require 'net/imap'
+require 'stringio'
+
+module Redwood
+
+class IMAP
+ attr_reader :uri
+ bool_reader :usual, :archived, :read, :dirty
+ attr_accessor :id, :labels
+
+ class Error < StandardError; end
+
+ def initialize uri, username, password, last_uid=nil, usual=true, archived=false, id=nil
+ raise "username and password must be specified" unless username && password
+
+ @uri_s = uri
+ @uri = URI(uri)
+ @username = username
+ @password = password
+ @last_uid = last_uid || 1
+ @dirty = false
+ @usual = usual
+ @archived = archived
+ @id = id
+ @imap = nil
+ @labels = [:unread,
+ archived ? nil : :inbox,
+ mailbox !~ /inbox/i && !mailbox.empty? ? mailbox.intern : nil,
+ ].compact
+ end
+
+ def connect
+ return if @imap
+ Redwood::log "connecting to #{@uri.host} port #{ssl? ? 993 : 143}, ssl=#{ssl?}"
+ #raise "simulated imap failure"
+ @imap = Net::IMAP.new @uri.host, ssl? ? 993 : 143, ssl?
+ @imap.authenticate('LOGIN', @username, @password)
+ Redwood::log "success. selecting #{mailbox.inspect}."
+ @imap.examine(mailbox)
+ end
+ private :connect
+
+ def mailbox; @uri.path[1..-1] end ##XXXX TODO handle nil
+ def ssl?; @uri.scheme == 'imaps' end
+ def reset!; @last_uid = 1; @dirty = true; end
+ def == o; o.is_a?(IMAP) && o.uri == uri; end
+ def uri; @uri.to_s; end
+ def to_s; uri; end
+ def is_source_for? s; to_s == s; end
+
+ def load_header uid=nil
+ MBox::read_header StringIO.new(raw_header(uid))
+ end
+
+ def load_message uid
+ RMail::Parser.read raw_full_message(uid)
+ end
+
+ ## load the full header text
+ def raw_header uid
+ connect
+ @imap.uid_fetch(uid, 'RFC822.HEADER')[0].attr['RFC822.HEADER'].gsub(/\r\n/, "\n")
+ end
+
+ def raw_full_message uid
+ connect
+ @imap.uid_fetch(uid, 'RFC822')[0].attr['RFC822'].gsub(/\r\n/, "\n")
+ end
+
+ def each
+ connect
+ uids = @imap.uid_search ['UID', "#{@last_uid}:#{total}"]
+ uids.each do |uid|
+ yield uid, labels
+ @last_uid = uid
+ @dirty = true
+ end
+ end
+
+ def done?; @last_uid >= total; end
+
+ def total
+ connect
+ @imap.uid_search(['ALL']).last
+ end
+end
+
+Redwood::register_yaml(IMAP, %w(uri_s username password last_uid usual archived id))
+
+end
raise "duplicate source!" if @sources.include? source
@sources_dirty = true
source.id ||= @sources.size
- source.id += 1 while @sources.member? source.id
+ ##TODO: why was this necessary?
+ ##source.id += 1 while @sources.member? source.id
@sources[source.id] = source
end
raise "no snippet" unless doc[:snippet]
begin
- Message.new source, doc[:source_info].to_i,
- doc[:label].split(" ").map { |s| s.intern },
- doc[:snippet]
+ Message.new :source => source, :source_info => doc[:source_info].to_i,
+ :labels => doc[:label].split(" ").map { |s| s.intern },
+ :snippet => doc[:snippet]
rescue MessageFormatError => e
raise IndexError.new(source, "error building message #{doc[:message_id]} at #{source}/#{doc[:source_info]}: #{e.message}")
-# rescue StandardError => e
+# rescue StandardError => e
# Message.new_from_index doc, <<EOS
# An error occurred while loading this message. It is possible that the source
# has changed, or (in the case of remote sources) is down. The error was:
def save_sources fn=Redwood::SOURCE_FN
if @sources_dirty || @sources.any? { |id, s| s.dirty? }
- FileUtils.mv fn, fn + ".bak", :force => true if File.exists? fn
+ bakfn = fn + ".bak"
+ if File.exists? fn
+ File.chmod 0600, fn
+ FileUtils.mv fn, bakfn, :force => true unless File.exists?(bakfn) && File.size(bakfn) > File.size(fn)
+ end
Redwood::save_yaml_obj @sources.values, fn
+ File.chmod 0600, fn
end
@sources_dirty = false
end
@filename == s || self.to_s == s
end
- def load_header offset=nil
+ def load_header offset
+ raise ArgumentError, "nil offset" unless offset
header = nil
@mutex.synchronize do
@f.seek offset if offset
l = @f.gets
- raise Error, "offset mismatch in mbox file: #{l.inspect}. Run 'sup-import --rebuild #{to_s}' to correct this." unless l =~ BREAK_RE
+ raise Error, "offset mismatch in mbox file offset #{offset.inspect}: #{l.inspect}. Run 'sup-import --rebuild #{to_s}' to correct this." unless l =~ BREAK_RE
header = MBox::read_header @f
end
header
BLOCK_QUOTE_PATTERN = /^-----\s*Original Message\s*----+$/
QUOTE_START_PATTERN = /(^\s*Excerpts from)|(^\s*In message )|(^\s*In article )|(^\s*Quoting )|((wrote|writes|said|says)\s*:\s*$)/
SIG_PATTERN = /(^-- ?$)|(^\s*----------+\s*$)|(^\s*_________+\s*$)/
- SIG_DISTANCE = 15 # lines from the end
+ MAX_SIG_DISTANCE = 15 # lines from the end
DEFAULT_SUBJECT = "(missing subject)"
DEFAULT_SENDER = "(missing sender)"
attr_reader :id, :date, :from, :subj, :refs, :replytos, :to, :source,
:cc, :bcc, :labels, :list_address, :recipient_email, :replyto,
- :source_info, :mbox_status
+ :source_info, :status
bool_reader :dirty
- def initialize source, source_info, labels, snippet=nil
- @source = source
- @source_info = source_info
+ ## if index_entry is specified, will fill in values from that,
+ def initialize opts
+ @source = opts[:source]
+ @source_info = opts[:source_info]
+ @snippet = opts[:snippet] || ""
+ @labels = opts[:labels] || []
@dirty = false
- @snippet = snippet
- @labels = labels
- header = @source.load_header @source_info
- header.each { |k, v| header[k.downcase] = v }
+ header =
+ if opts[:header]
+ opts[:header]
+ else
+ header = @source.load_header @source_info
+ header.each { |k, v| header[k.downcase] = v }
+ header
+ end
%w(message-id date).each do |f|
raise MessageFormatError, "no #{f} field in header #{header.inspect} (source #@source offset #@source_info)" unless header.include? f
end
@recipient_email = header["delivered-to"]
- @mbox_status = header["status"]
- end
-
- def snippet
- to_chunks unless @snippet
- @snippet
+ @status = header["status"]
end
+ def snippet; @snippet || to_chunks && @snippet; end
def is_list_message?; !@list_address.nil?; end
def is_draft?; DraftLoader === @source; end
def draft_filename
newstate = nil
if line =~ QUOTE_PATTERN || (line =~ QUOTE_START_PATTERN && (nextline =~ QUOTE_PATTERN || nextline =~ QUOTE_START_PATTERN))
newstate = :quote
- elsif line =~ SIG_PATTERN && (lines.length - i) < SIG_DISTANCE
+ elsif line =~ SIG_PATTERN && (lines.length - i) < MAX_SIG_DISTANCE
newstate = :sig
elsif line =~ BLOCK_QUOTE_PATTERN
newstate = :block_quote
newstate = nil
if line =~ QUOTE_PATTERN || line =~ QUOTE_START_PATTERN || line =~ /^\s*$/
chunk_lines << line
- elsif line =~ SIG_PATTERN && (lines.length - i) < SIG_DISTANCE
+ elsif line =~ SIG_PATTERN && (lines.length - i) < MAX_SIG_DISTANCE
newstate = :sig
else
newstate = :text
@header.delete "Date"
@header["Message-Id"] = gen_message_id # generate a new'n
regen_text
+ @sent = false
end
def send_message
- DraftManager.discard @id if super
+ if super
+ DraftManager.discard @id
+ @sent = true
+ end
+ end
+
+ def cleanup
+ unless @sent
+ if BufferManager.ask_yes_or_no "discard draft?"
+ DraftManager.discard @id
+ BufferManager.flash "Draft discarded."
+ else
+ BufferManager.flash "Draft saved."
+ end
+ super
+ end
end
end
threads = @threads + @hidden_threads.keys
mbid = BufferManager.say "Saving threads..."
threads.each_with_index do |t, i|
- BufferManager.say "Saving thread #{i + 1} of #{threads.length}...",
- mbid
+ if i % 5 == 0
+ BufferManager.say "Saving thread #{i + 1} of #{threads.length}...",
+ mbid
+ end
t.save Index
end
BufferManager.clear mbid
start_offset ||= offset
yield " Found message at #{offset} with labels #{labels * ', '}"
begin
- m = Redwood::Message.new source, offset, labels
+ m = Redwood::Message.new :source => source, :source_info => offset,
+ :labels => labels
if found[m.id]
- yield "Skipping duplicate message #{m.id}"
+ yield "Skipping duplicate message #{m.id} (source total #{source.total})"
next
else
found[m.id] = true
def write_sent_message date, from_email
need_blank = File.exists?(@fn) && !File.zero?(@fn)
File.open(@fn, "a") do |f|
- if need_blank
- @source.increment_offset if @source.offset == f.tell
- f.puts
- end
+ f.puts if need_blank
f.puts "From #{from_email} #{date}"
yield f
end
@source.each do |offset, labels|
- m = Message.new @source, offset, labels
+ m = Message.new :source => @source, :source_info => offset, :labels => labels
Index.add_message m
UpdateManager.relay :add, m
end
super filename, end_offset, true, true
end
- def increment_offset; @end_offset += 1; end
- def offset; @end_offset; end
def id; SentManager.source_id; end
def to_s; SentManager.source_name; end
projects / sup / commitdiff