lib/sup/keymap.rb
lib/sup/label.rb
lib/sup/logger.rb
+lib/sup/interactive-lock.rb
lib/sup/maildir.rb
lib/sup/mbox.rb
lib/sup/mbox/loader.rb
lib/sup/rfc2047.rb
lib/sup/sent.rb
lib/sup/source.rb
-lib/sup/suicide.rb
lib/sup/tagger.rb
lib/sup/textfield.rb
lib/sup/thread.rb
k.add :compose, "Compose new message", 'm', 'c'
k.add :nothing, "Do nothing", :ctrl_g
k.add :recall_draft, "Edit most recent draft message", 'R'
+ k.add :show_inbox, "Show the Inbox buffer", 'I'
+ k.add :show_console, "Show the Console buffer", '~'
end
## the following magic enables wide characters when used with a ruby
module_function :start_cursing, :stop_cursing
Index.init
-begin
- Index.lock
-rescue Index::LockError => e
- require 'highline'
-
- h = HighLine.new
- h.wrap_at = :auto
- h.say Index.fancy_lock_error_message_for(e)
-
- case h.ask("Should I ask that process to kill itself? ")
- when /^\s*y(es)?\s*$/i
- h.say "Ok, suggesting seppuku..."
- FileUtils.touch Redwood::SUICIDE_FN
- sleep SuicideManager::DELAY * 2
- FileUtils.rm_f Redwood::SUICIDE_FN
- h.say "Let's try that again."
- retry
- else
- h.say <<EOS
-Ok, giving up. If the process crashed and left a stale lockfile, you
-can fix this by manually deleting #{Index.lockfile}.
-EOS
- exit
- end
-end
+Index.lock_interactively or exit
begin
Redwood::start
Index.load
+ $die = false
+ trap("TERM") { |x| $die = true }
+ trap("WINCH") { |x| BufferManager.sigwinch_happened! }
+
if(s = Redwood::SourceManager.source_for DraftManager.source_name)
DraftManager.source = s
else
unless $opts[:no_threads]
PollManager.start
- SuicideManager.start
Index.start_lock_update_thread
end
SearchResultsMode.spawn_from_query $opts[:search]
end
- until Redwood::exceptions.nonempty? || SuicideManager.die?
- c =
- begin
- Ncurses.nonblocking_getch
- rescue Exception => e
- if e.is_a?(Interrupt)
- raise if BufferManager.ask_yes_or_no("Die ungracefully now?")
- bm.draw_screen
- nil
- end
- end
- next unless c
+ until Redwood::exceptions.nonempty? || $die
+ c = begin
+ Ncurses.nonblocking_getch
+ rescue Interrupt => e
+ raise if BufferManager.ask_yes_or_no "Die ungracefully now?"
+ BufferManager.draw_screen
+ nil
+ end
+
+ if c.nil?
+ if BufferManager.sigwinch_happened?
+ debug "redrawing screen on sigwinch"
+ BufferManager.completely_redraw_screen
+ end
+ next
+ end
+
+ if c == 410
+ ## this is ncurses's way of telling us it's detected a refresh.
+ ## since we have our own sigwinch handler, we don't do anything.
+ next
+ end
+
bm.erase_flash
action =
b, new = BufferManager.spawn_unless_exists("All drafts") { LabelSearchResultsMode.new [:draft] }
b.mode.load_threads :num => b.content_height if new
end
+ when :show_inbox
+ BufferManager.raise_to_front ibuf
+ when :show_console
+ b, new = bm.spawn_unless_exists("Console", :system => true) { ConsoleMode.new }
+ b.mode.run
when :nothing, InputSequenceAborted
when :redraw
bm.completely_redraw_screen
bm.draw_screen
end
- bm.kill_all_buffers if SuicideManager.die?
+ bm.kill_all_buffers if $die
rescue Exception => e
Redwood::record_exception e, "main"
ensure
unless $opts[:no_threads]
PollManager.stop if PollManager.instantiated?
- SuicideManager.stop if PollManager.instantiated?
Index.stop_lock_update_thread
end
Redwood::Logger.add_sink $stderr, false
debug "stopped cursing"
- if SuicideManager.instantiated? && SuicideManager.die?
+ if $die
info "I've been ordered to commit seppuku. I obey!"
end
Redwood::start
index = Redwood::Index.init
-index.lock_or_die
+index.lock_interactively or exit
begin
Redwood::SourceManager.load_sources
end
seen = {}
-index.lock_or_die
+index.lock_interactively or exit
begin
index.load
Redwood::start
index = Redwood::Index.init
-index.lock_or_die
+index.lock_interactively or exit
deleted_fp, spam_fp = nil
unless opts[:dry_run]
remove_labels = opts[:remove].to_set_of_symbols ","
Trollop::die "nothing to do: no labels to add or remove" if add_labels.empty? && remove_labels.empty?
+Trollop::die "no sources specified" if ARGV.empty?
Redwood::start
+index = Redwood::Index.init
+index.lock_interactively or exit
begin
- index = Redwood::Index.init
index.load
source_ids = if opts[:all_sources]
module_function :reporting_thread, :record_exception, :exceptions
## one-stop shop for yamliciousness
- def save_yaml_obj object, fn, safe=false
+ def save_yaml_obj o, fn, safe=false
+ o = if o.is_a?(Array)
+ o.map { |x| (x.respond_to?(:before_marshal) && x.before_marshal) || x }
+ else
+ o.respond_to?(:before_marshal) && o.before_marshal
+ end
+
if safe
safe_fn = "#{File.dirname fn}/safe_#{File.basename fn}"
mode = File.stat(fn).mode if File.exists? fn
- File.open(safe_fn, "w", mode) { |f| f.puts object.to_yaml }
+ File.open(safe_fn, "w", mode) { |f| f.puts o.to_yaml }
FileUtils.mv safe_fn, fn
else
- File.open(fn, "w") { |f| f.puts object.to_yaml }
+ File.open(fn, "w") { |f| f.puts o.to_yaml }
end
end
def load_yaml_obj fn, compress=false
- if File.exists? fn
+ o = if File.exists? fn
if compress
Zlib::GzipReader.open(fn) { |f| YAML::load f }
else
YAML::load_file fn
end
end
+ if o.is_a?(Array)
+ o.each { |x| x.after_unmarshal! if x.respond_to?(:after_unmarshal!) }
+ else
+ o.after_unmarshal! if o.respond_to?(:after_unmarshal!)
+ end
+ o
end
def start
Redwood::DraftManager.init Redwood::DRAFT_DIR
Redwood::UpdateManager.init
Redwood::PollManager.init
- Redwood::SuicideManager.init Redwood::SUICIDE_FN
Redwood::CryptoManager.init
Redwood::UndoManager.init
Redwood::SourceManager.init
require "sup/modes/text-mode"
require "sup/modes/log-mode"
require "sup/update"
-require "sup/suicide"
require "sup/message-chunks"
require "sup/message"
require "sup/source"
require "sup/person"
require "sup/account"
require "sup/thread"
+require "sup/interactive-lock"
require "sup/index"
require "sup/textfield"
require "sup/colormap"
require "sup/modes/poll-mode"
require "sup/modes/file-browser-mode"
require "sup/modes/completion-mode"
+require "sup/modes/console-mode"
require "sup/sent"
$:.each do |base|
## magically, this stuff seems to work now. i could swear it didn't
## before. hm.
def nonblocking_getch
- if IO.select([$stdin], nil, nil, 1)
- Ncurses.getch
- else
- nil
+ ## INSANTIY
+ ## it is NECESSARY to wrap Ncurses.getch in a select() otherwise all
+ ## background threads will be BLOCKED. (except in very modern versions
+ ## of libncurses-ruby. the current one on ubuntu seems to work well.)
+ if IO.select([$stdin], nil, nil, 0.5)
+ c = Ncurses.getch
end
end
@flash = nil
@shelled = @asking = false
@in_x = ENV["TERM"] =~ /(xterm|rxvt|screen)/
+ @sigwinch_happened = false
+ @sigwinch_mutex = Mutex.new
end
+ def sigwinch_happened!; @sigwinch_mutex.synchronize { @sigwinch_happened = true } end
+ def sigwinch_happened?; @sigwinch_mutex.synchronize { @sigwinch_happened } end
+
def buffers; @name_map.to_a; end
def focus_on buf
## have to change this. but it's not clear that we will ever actually
## do that.
def roll_buffers
- @buffers.last.force_to_top = false
- raise_to_front @buffers.first
+ bufs = rollable_buffers
+ bufs.last.force_to_top = false
+ raise_to_front bufs.first
end
def roll_buffers_backwards
- return unless @buffers.length > 1
- @buffers.last.force_to_top = false
- raise_to_front @buffers[@buffers.length - 2]
+ bufs = rollable_buffers
+ return unless bufs.length > 1
+ bufs.last.force_to_top = false
+ raise_to_front bufs[bufs.length - 2]
+ end
+
+ def rollable_buffers
+ @buffers.select { |b| !b.system? || @buffers.last == b }
end
def handle_input c
def completely_redraw_screen
return if @shelled
+ ## this magic makes Ncurses get the new size of the screen
+ Ncurses.endwin
+ Ncurses.stdscr.keypad 1
+ Ncurses.curs_set 0
+ Ncurses.refresh
+ @sigwinch_mutex.synchronize { @sigwinch_happened = false }
+ debug "new screen size is #{Ncurses.rows} x #{Ncurses.cols}"
+
status, title = get_status_and_title(@focus_buf) # must be called outside of the ncurses lock
Ncurses.sync do
class FerretIndex < BaseIndex
+ HookManager.register "custom-search", <<EOS
+Executes before a string search is applied to the index,
+returning a new search string.
+Variables:
+ subs: The string being searched. Be careful about shadowing:
+ this variable is actually a method, so use a temporary variable
+ or explicitly call self.subs; the substitutions in index.rb
+ don't actually work.
+EOS
+
def initialize dir=BASE_DIR
super
def parse_query s
query = {}
- subs = s.gsub(/\b(to|from):(\S+)\b/) do
+ subs = HookManager.run("custom-search", :subs => s) || s
+
+ subs = subs.gsub(/\b(to|from):(\S+)\b/) do
field, name = $1, $2
if(p = ContactManager.contact_for(name))
[field, p.email]
attr_writer :__locals
+ ## an annoying gotcha here is that if you try something
+ ## like var = var.foo(), var will magically get allocated
+ ## to Nil and method_missing will never get called. You
+ ## can work around this by calling self.var or simply
+ ## not assigning it to itself.
def method_missing m, *a
case @__locals[m]
when Proc
def enabled? name; !hook_for(name).nil? end
+ def clear; @hooks.clear; end
+
private
def hook_for name
module Redwood
class IMAP < Source
+ include SerializeLabelsNicely
SCAN_INTERVAL = 60 # seconds
## upon these errors we'll try to rereconnect a few times
module Redwood
class BaseIndex
+ include InteractiveLock
+
class LockError < StandardError
def initialize h
@h = h
@lock_update_thread = nil
end
- def possibly_pluralize number_of, kind
- "#{number_of} #{kind}" +
- if number_of == 1 then "" else "s" end
- end
-
- def fancy_lock_error_message_for e
- secs = (Time.now - e.mtime).to_i
- mins = secs / 60
- time =
- if mins == 0
- possibly_pluralize secs , "second"
- else
- possibly_pluralize mins, "minute"
- end
-
- <<EOS
-Error: the sup index is locked by another process! User '#{e.user}' on
-host '#{e.host}' is running #{e.pname} with pid #{e.pid}. The process was alive
-as of #{time} ago.
-EOS
- end
-
- def lock_or_die
- begin
- lock
- rescue LockError => e
- $stderr.puts fancy_lock_error_message_for(e)
- $stderr.puts <<EOS
-
-You can wait for the process to finish, or, if it crashed and left a
-stale lock file behind, you can manually delete #{@lock.path}.
-EOS
- exit
- end
- end
-
def unlock
if @lock && @lock.locked?
debug "unlocking #{lockfile}..."
--- /dev/null
+require 'fileutils'
+
+module Redwood
+
+## wrap a nice interactive layer on top of anything that has a #lock method
+## which throws a LockError which responds to #user, #host, #mtim, #pname, and
+## #pid.
+
+module InteractiveLock
+ def pluralize number_of, kind; "#{number_of} #{kind}" + (number_of == 1 ? "" : "s") end
+
+ def time_ago_in_words time
+ secs = (Time.now - time).to_i
+ mins = secs / 60
+ time = if mins == 0
+ pluralize secs, "second"
+ else
+ pluralize mins, "minute"
+ end
+ end
+
+ DELAY = 5 # seconds
+
+ def lock_interactively stream=$stderr
+ begin
+ Index.lock
+ rescue Index::LockError => e
+ stream.puts <<EOS
+Error: the index is locked by another process! User '#{e.user}' on
+host '#{e.host}' is running #{e.pname} with pid #{e.pid}.
+The process was alive as of at least #{time_ago_in_words e.mtime} ago.
+
+EOS
+ stream.print "Should I ask that process to kill itself (y/n)? "
+ stream.flush
+
+ success = if $stdin.gets =~ /^\s*y(es)?\s*$/i
+ stream.puts "Ok, trying to kill process..."
+
+ begin
+ Process.kill "TERM", e.pid.to_i
+ sleep DELAY
+ rescue Errno::ESRCH # no such process
+ stream.puts "Hm, I couldn't kill it."
+ end
+
+ stream.puts "Let's try that again."
+ begin
+ Index.lock
+ rescue Index::LockError => e
+ stream.puts "I couldn't lock the index. The lockfile might just be stale."
+ stream.print "Should I just remove it and continue? (y/n) "
+ stream.flush
+
+ if $stdin.gets =~ /^\s*y(es)?\s*$/i
+ FileUtils.rm e.path
+
+ stream.puts "Let's try that one more time."
+ begin
+ Index.lock
+ true
+ rescue Index::LockError => e
+ end
+ end
+ end
+ end
+
+ stream.puts "Sorry, couldn't unlock the index." unless success
+ success
+ end
+ end
+end
+
+end
## pathnames on disk.
class Maildir < Source
+ include SerializeLabelsNicely
SCAN_INTERVAL = 30 # seconds
MYHOSTNAME = Socket.gethostname
def maildir_data msg
fn = File.basename @ids_to_fns[msg]
- fn =~ %r{^([^:,]+):([12]),([DFPRST]*)$}
+ fn =~ %r{^([^:]+):([12]),([DFPRST]*)$}
[($1 || fn), ($2 || "2"), ($3 || "")]
end
module MBox
class Loader < Source
+ include SerializeLabelsNicely
yaml_properties :uri, :cur_offset, :usual, :archived, :id, :labels
## uri_or_fp is horrific. need to refactor.
def initial_state; :open end
def viewable?; @lines.nil? end
def view_default! path
- cmd = "/usr/bin/run-mailcap --action=view '#{@content_type}:#{path}' 2>/dev/null"
+ cmd = "/usr/bin/run-mailcap --action=view '#{@content_type}:#{path}'"
debug "running: #{cmd.inspect}"
- system cmd
+ BufferManager.shell_out(cmd)
$? == 0
end
class EnclosedMessage
attr_reader :lines
- def initialize from, body
- @from = from
- @lines = body.split "\n"
- end
+ def initialize from, to, cc, date, subj
+ @from = from ? "unknown sender" : from.full_adress
+ @to = to ? "" : to.map { |p| p.full_address }.join(", ")
+ @cc = cc ? "" : cc.map { |p| p.full_address }.join(", ")
+ if date
+ @date = date.rfc822
+ else
+ @date = ""
+ end
- def from
- @from ? @from.longname : "unknown sender"
+ @subj = subj
+
+ @lines = "\nFrom: #{from}\n"
+ @lines += "To: #{to}\n"
+ if !cc.empty?
+ @lines += "Cc: #{cc}\n"
+ end
+ @lines += "Date: #{date}\n"
+ @lines += "Subject: #{subj}\n\n"
end
def inlineable?; false end
def viewable?; false end
def patina_color; :generic_notice_patina_color end
- def patina_text; "Begin enclosed message from #{from} (#{@lines.length} lines)" end
+ def patina_text; "Begin enclosed message sent on #{@date}" end
def color; :quote_color end
end
chunks
elsif m.header.content_type == "message/rfc822"
payload = RMail::Parser.read(m.body)
- from = payload.header.from.first
- from_person = from ? Person.from_address(from.format) : nil
- [Chunk::EnclosedMessage.new(from_person, payload.to_s)] +
- message_to_chunks(payload, encrypted)
+ from = payload.header.from.first ? payload.header.from.first.format : ""
+ to = payload.header.to.map { |p| p.format }.join(", ")
+ cc = payload.header.cc.map { |p| p.format }.join(", ")
+ subj = payload.header.subject
+ subj = subj ? Message.normalize_subj(payload.header.subject.gsub(/\s+/, " ").gsub(/\s+$/, "")) : subj
+ if Rfc2047.is_encoded? subj
+ subj = Rfc2047.decode_to $encoding, subj
+ end
+ msgdate = payload.header.date
+ from_person = from ? Person.from_address(from) : nil
+ to_people = to ? Person.from_address_list(to) : nil
+ cc_people = cc ? Person.from_address_list(cc) : nil
+ [Chunk::EnclosedMessage.new(from_person, to_people, cc_people, msgdate, subj)] + message_to_chunks(payload, encrypted)
else
filename =
## first, paw through the headers looking for a filename
--- /dev/null
+require 'pp'
+
+module Redwood
+
+class Console
+ def initialize mode
+ @mode = mode
+ end
+
+ def query(query)
+ Enumerable::Enumerator.new(Index, :each_message, Index.parse_query(query))
+ end
+
+ def add_labels(query, *labels)
+ query(query).each { |m| m.labels += labels; m.save Index }
+ end
+
+ def remove_labels(query, *labels)
+ query(query).each { |m| m.labels -= labels; m.save Index }
+ end
+
+ def xapian; Index.instance.instance_variable_get :@xapian; end
+ def ferret; Index.instance.instance_variable_get :@index; end
+
+ ## files that won't cause problems when reloaded
+ ## TODO expand this list / convert to blacklist
+ RELOAD_WHITELIST = %w(sup/xapian_index.rb sup/modes/console-mode.rb)
+
+ def reload
+ old_verbose = $VERBOSE
+ $VERBOSE = nil
+ old_features = $".dup
+ begin
+ fs = $".grep(/^sup\//)
+ fs.reject! { |f| not RELOAD_WHITELIST.member? f }
+ fs.each { |f| $".delete f }
+ fs.each do |f|
+ @mode << "reloading #{f}\n"
+ begin
+ require f
+ rescue LoadError => e
+ raise unless e.message =~ /no such file to load/
+ end
+ end
+ rescue Exception
+ $".clear
+ $".concat old_features
+ raise
+ ensure
+ $VERBOSE = old_verbose
+ end
+ true
+ end
+
+ def clear_hooks
+ HookManager.clear
+ nil
+ end
+end
+
+class ConsoleMode < LogMode
+ register_keymap do |k|
+ k.add :run, "Restart evaluation", 'e'
+ end
+
+ def initialize
+ super "console"
+ @console = Console.new self
+ @binding = @console.instance_eval { binding }
+ self << <<EOS
+Sup #{VERSION} console.
+Available commands: #{(@console.methods - Object.methods) * ", "}
+Ctrl-g stops evaluation; 'e' restarts it.
+
+EOS
+ end
+
+ def execute cmd
+ begin
+ self << ">> #{cmd}\n"
+ ret = eval cmd, @binding
+ self << "=> #{ret.pretty_inspect}\n"
+ rescue Exception
+ self << "#{$!.class}: #{$!.message}\n"
+ clean_backtrace = []
+ $!.backtrace.each { |l| break if l =~ /console-mode/; clean_backtrace << l }
+ clean_backtrace.each { |l| self << "#{l}\n" }
+ end
+ end
+
+ def prompt
+ BufferManager.ask :console, "eval: "
+ end
+
+ def run
+ while true
+ cmd = prompt or return
+ execute cmd
+ end
+ end
+end
+
+end
end
end
+## if you have a @labels instance variable, include this
+## to serialize them nicely as an array, rather than as a
+## nasty set.
+module SerializeLabelsNicely
+ def before_marshal # can return an object
+ c = clone
+ c.instance_eval { @labels = @labels.to_a.map { |l| l.to_s } }
+ c
+ end
+
+ def after_unmarshal!
+ @labels = Set.new(@labels.map { |s| s.to_sym })
+ end
+end
+
class SourceManager
include Singleton
File.chmod 0600, fn
FileUtils.mv fn, bakfn, :force => true unless File.exists?(bakfn) && File.size(fn) == 0
end
- Redwood::save_yaml_obj sources.sort_by { |s| s.id.to_i }, fn, true
+ Redwood::save_yaml_obj sources, fn, true
File.chmod 0600, fn
end
@sources_dirty = false
+++ /dev/null
-module Redwood
-
-class SuicideManager
- include Singleton
-
- DELAY = 5
-
- def initialize fn
- @fn = fn
- @die = false
- @thread = nil
- FileUtils.rm_f @fn
- end
-
- bool_reader :die
-
- def start
- @thread = Redwood::reporting_thread("suicide watch") do
- while true
- sleep DELAY
- if File.exists? @fn
- FileUtils.rm_f @fn
- @die = true
- end
- end
- end
- end
-
- def stop
- @thread.kill if @thread
- @thread = nil
- end
-end
-
-end
def lockinfo_on_disk
h = load_lock_id IO.read(path)
h['mtime'] = File.mtime path
+ h['path'] = path
h
end
class Range
## only valid for integer ranges (unless I guess it's exclusive)
- def size
+ def size
last - first + (exclude_end? ? 0 : 1)
end
end
require 'xapian'
-require 'gdbm'
require 'set'
module Redwood
# for searching due to precomputing thread membership.
class XapianIndex < BaseIndex
STEM_LANGUAGE = "english"
+ INDEX_VERSION = '1'
## dates are converted to integers for xapian, and are used for document ids,
## so we must ensure they're reasonably valid. this typically only affect
end
def load_index
- @entries = MarshalledGDBM.new File.join(@dir, "entries.db")
- @docids = MarshalledGDBM.new File.join(@dir, "docids.db")
- @thread_members = MarshalledGDBM.new File.join(@dir, "thread_members.db")
- @thread_ids = MarshalledGDBM.new File.join(@dir, "thread_ids.db")
- @assigned_docids = GDBM.new File.join(@dir, "assigned_docids.db")
-
- @xapian = Xapian::WritableDatabase.new(File.join(@dir, "xapian"), Xapian::DB_CREATE_OR_OPEN)
+ path = File.join(@dir, 'xapian')
+ if File.exists? path
+ @xapian = Xapian::WritableDatabase.new(path, Xapian::DB_OPEN)
+ db_version = @xapian.get_metadata 'version'
+ db_version = '0' if db_version.empty?
+ if db_version != INDEX_VERSION
+ fail "This Sup version expects a v#{INDEX_VERSION} index, but you have an existing v#{db_version} index. Please downgrade to your previous version and dump your labels before upgrading to this version (then run sup-sync --restore)."
+ end
+ else
+ @xapian = Xapian::WritableDatabase.new(path, Xapian::DB_CREATE)
+ @xapian.set_metadata 'version', INDEX_VERSION
+ end
@term_generator = Xapian::TermGenerator.new()
@term_generator.stemmer = Xapian::Stem.new(STEM_LANGUAGE)
@enquire = Xapian::Enquire.new @xapian
end
def contains_id? id
- synchronize { @entries.member? id }
+ synchronize { find_docid(id) && true }
end
def source_for_id id
- synchronize { @entries[id][:source_id] }
+ synchronize { get_entry(id)[:source_id] }
end
def delete id
- synchronize { @xapian.delete_document @docids[id] }
+ synchronize { @xapian.delete_document mkterm(:msgid, id) }
end
def build_message id
- entry = synchronize { @entries[id] }
+ entry = synchronize { get_entry id }
return unless entry
source = SourceManager[entry[:source_id]]
def update_message_state m; sync_message m end
def sync_message m, opts={}
- entry = synchronize { @entries[m.id] }
+ entry = synchronize { get_entry m.id }
snippet = m.snippet
entry ||= {}
labels = m.labels
labels.each { |l| LabelManager << l }
synchronize do
- index_message m, opts
- union_threads([m.id] + m.refs + m.replytos)
- @entries[m.id] = d
+ index_message m, d, opts
end
true
end
def each_message_in_thread_for m, opts={}
# TODO thread by subject
# TODO handle killed threads
- ids = synchronize { @thread_members[@thread_ids[m.id]] } || []
- ids.select { |id| contains_id? id }.each { |id| yield id, lambda { build_message id } }
+ return unless doc = find_doc(m.id)
+ queue = doc.value(THREAD_VALUENO).split(',')
+ msgids = [m.id]
+ seen_threads = Set.new
+ seen_messages = Set.new [m.id]
+ while not queue.empty?
+ thread_id = queue.pop
+ next if seen_threads.member? thread_id
+ return false if thread_killed? thread_id
+ seen_threads << thread_id
+ docs = term_docids(mkterm(:thread, thread_id)).map { |x| @xapian.document x }
+ docs.each do |doc|
+ msgid = doc.value MSGID_VALUENO
+ next if seen_messages.member? msgid
+ msgids << msgid
+ seen_messages << msgid
+ queue.concat doc.value(THREAD_VALUENO).split(',')
+ end
+ end
+ msgids.each { |id| yield id, lambda { build_message id } }
true
end
'label' => 'L',
'source_id' => 'I',
'attachment_extension' => 'O',
+ 'msgid' => 'Q',
+ 'thread' => 'H',
+ 'ref' => 'R',
}
PREFIX = NORMAL_PREFIX.merge BOOLEAN_PREFIX
- DATE_VALUENO = 0
+ MSGID_VALUENO = 0
+ THREAD_VALUENO = 1
+ DATE_VALUENO = 2
MAX_TERM_LENGTH = 245
def assign_docid m, truncated_date
t = (truncated_date.to_i - MIDDLE_DATE.to_i).to_f
docid = (DOCID_SCALE - DOCID_SCALE/(Math::E**(-(t/TIME_SCALE)) + 1)).to_i
+ while docid > 0 and docid_exists? docid
+ docid -= 1
+ end
+ docid > 0 ? docid : nil
+ end
+
+ # XXX is there a better way?
+ def docid_exists? docid
begin
- while @assigned_docids.member? [docid].pack("N")
- docid -= 1
- end
- rescue
+ @xapian.doclength docid
+ true
+ rescue RuntimeError #Xapian::DocNotFoundError
+ raise unless $!.message =~ /DocNotFoundError/
+ false
end
- @assigned_docids[[docid].pack("N")] = ''
- docid
+ end
+
+ def term_docids term
+ @xapian.postlist(term).map { |x| x.docid }
+ end
+
+ def find_docid id
+ term_docids(mkterm(:msgid,id)).tap { |x| fail unless x.size <= 1 }.first
+ end
+
+ def find_doc id
+ return unless docid = find_docid(id)
+ @xapian.document docid
+ end
+
+ def get_id docid
+ return unless doc = @xapian.document(docid)
+ doc.value MSGID_VALUENO
+ end
+
+ def get_entry id
+ return unless doc = find_doc(id)
+ Marshal.load doc.data
+ end
+
+ def thread_killed? thread_id
+ not run_query(Q.new(Q::OP_AND, mkterm(:thread, thread_id), mkterm(:label, :Killed)), 0, 1).empty?
end
def synchronize &b
def run_query_ids xapian_query, offset, limit
matchset = run_query xapian_query, offset, limit
- matchset.matches.map { |r| r.document.data }
+ matchset.matches.map { |r| r.document.value MSGID_VALUENO }
end
Q = Xapian::Query
end
end
- def index_message m, opts
+ def index_message m, entry, opts
terms = []
text = []
terms << mkterm(:date,m.date) if m.date
m.labels.each { |t| terms << mkterm(:label,t) }
terms << mkterm(:type, 'mail')
+ terms << mkterm(:msgid, m.id)
terms << mkterm(:source_id, m.source.id)
m.attachments.each do |a|
a =~ /\.(\w+)$/ or next
terms << t
end
+ ## Thread membership
+ children = term_docids(mkterm(:ref, m.id)).map { |docid| @xapian.document docid }
+ parent_ids = m.refs + m.replytos
+ parents = parent_ids.map { |id| find_doc id }.compact
+ thread_members = SavingHash.new { [] }
+ (children + parents).each do |doc2|
+ thread_ids = doc2.value(THREAD_VALUENO).split ','
+ thread_ids.each { |thread_id| thread_members[thread_id] << doc2 }
+ end
+
+ thread_ids = thread_members.empty? ? [m.id] : thread_members.keys
+
+ thread_ids.each { |thread_id| terms << mkterm(:thread, thread_id) }
+ parent_ids.each do |ref|
+ terms << mkterm(:ref, ref)
+ end
+
# Full text search content
text << [subject_text, PREFIX['subject']]
text << [subject_text, PREFIX['body']]
Xapian.sortable_serialise 0
end
- doc = Xapian::Document.new
- docid = @docids[m.id] || assign_docid(m, truncated_date)
+ docid = nil
+ unless doc = find_doc(m.id)
+ doc = Xapian::Document.new
+ if not docid = assign_docid(m, truncated_date)
+ # Could be triggered by spam
+ Redwood::log "warning: docid underflow, dropping #{m.id.inspect}"
+ return
+ end
+ else
+ doc.clear_terms
+ doc.clear_values
+ docid = doc.docid
+ end
@term_generator.document = doc
text.each { |text,prefix| @term_generator.index_text text, 1, prefix }
terms.each { |term| doc.add_term term if term.length <= MAX_TERM_LENGTH }
+ doc.add_value MSGID_VALUENO, m.id
+ doc.add_value THREAD_VALUENO, (thread_ids * ',')
doc.add_value DATE_VALUENO, date_value
- doc.data = m.id
+ doc.data = Marshal.dump entry
@xapian.replace_document docid, doc
- @docids[m.id] = docid
end
# Construct a Xapian term
PREFIX['source_id'] + args[0].to_s.downcase
when :attachment_extension
PREFIX['attachment_extension'] + args[0].to_s.downcase
+ when :msgid, :ref, :thread
+ PREFIX[type.to_s] + args[0][0...(MAX_TERM_LENGTH-1)]
else
raise "Invalid term type #{type}"
end
end
- # Join all the given message-ids into a single thread
- def union_threads ids
- seen_threads = Set.new
- related = Set.new
-
- # Get all the ids that will be in the new thread
- ids.each do |id|
- related << id
- thread_id = @thread_ids[id]
- if thread_id && !seen_threads.member?(thread_id)
- thread_members = @thread_members[thread_id]
- related.merge thread_members
- seen_threads << thread_id
- end
- end
-
- # Pick a leader and move all the others to its thread
- a = related.to_a
- best, *rest = a.sort_by { |x| x.hash }
- @thread_members[best] = a
- @thread_ids[best] = best
- rest.each do |x|
- @thread_members.delete x
- @thread_ids[x] = best
- end
- end
end
end
-
-class MarshalledGDBM < GDBM
- def []= k, v
- super k, Marshal.dump(v)
- end
-
- def [] k
- v = super k
- v ? Marshal.load(v) : nil
- end
-end
projects / sup / commitdiff