From: wmorgan Date: Thu, 7 Dec 2006 21:35:26 +0000 (+0000) Subject: preparations for imap X-Git-Url: https://git.notmuchmail.org/git?a=commitdiff_plain;h=1b3171df2187359537d03998a680b52eec23aed4;p=sup preparations for imap git-svn-id: svn://rubyforge.org/var/svn/sup/trunk@66 5c8cc53c-5e98-4d25-b20a-d8db53a31250 --- diff --git a/bin/sup b/bin/sup index c5e54fa..f8e50b7 100644 --- a/bin/sup +++ b/bin/sup @@ -109,7 +109,7 @@ begin bm.draw_screen imode.load_more_threads ibuf.content_height - ::Thread.new { sleep 5; PollManager.poll } + ::Thread.new { sleep 3; PollManager.poll } until $exception bm.draw_screen @@ -217,6 +217,4 @@ EOS raise $exception end - end - diff --git a/bin/sup-import b/bin/sup-import index c135636..c7fe2ad 100644 --- a/bin/sup-import +++ b/bin/sup-import @@ -82,7 +82,6 @@ end puts "loading index..." index = Redwood::Index.new index.load -pre_nm = index.size puts "loaded index of #{index.size} messages" sources = ARGV.map do |fn| @@ -126,7 +125,7 @@ begin labels -= [:inbox] if force_archive labels -= [:unread] if force_read begin - m = Redwood::Message.new source, offset, labels + m = Redwood::Message.new :source => source, :source_info => offset, :labels => labels if found[m.id] puts "skipping duplicate message #{m.id}" next @@ -145,7 +144,7 @@ begin else num += 1 if index.add_message m end - rescue Redwood::MessageFormatError => e + rescue Redwood::MessageFormatError, Redwood::MBox::Error => e $stderr.puts "ignoring erroneous message at #{source}##{offset}: #{e.message}" end if num % 1000 == 0 && num > 0 diff --git a/bin/sup-recover-sources b/bin/sup-recover-sources new file mode 100644 index 0000000..af39b7d --- /dev/null +++ b/bin/sup-recover-sources @@ -0,0 +1,100 @@ +#!/usr/bin/env ruby + +require 'optparse' + +$opts = { + :unusual => false, + :archive => false, + :scan_num => 10, +} + + +OPTIONPARSERSUCKS = "\n" + " " * 38 +OptionParser.new do |opts| + opts.banner = <+ + +Rebuilds a lost sources.yaml file by reading messages from a list of +sources and determining, for each source, the most prevalent +'source_id' field of messages from that source in the index. + +The only non-deterministic component to this is that if the same +message appears in multiple sources, those sources may be +mis-diagnosed by this program. + +If the first N messages (--scan-num below) all have the same source_id +in the index, the source will be added to sources.yaml. Otherwise, the +distribution will be printed, and you will have to add it by hand. + +The offset pointer into the sources will be set to the end of the source, +so you will have to run sup-import --rebuild for each new source after +doing this. + +Options include: +EOS + + opts.on("--unusual", "Mark sources as 'unusual'. Only usual#{OPTIONPARSERSUCKS}sources will be polled by hand. Default:#{OPTIONPARSERSUCKS}#{$opts[:unusual]}.") { $opts[:unusual] = true } + + opts.on("--archive", "Mark sources as 'archive'. New messages#{OPTIONPARSERSUCKS}from these sources will not appear in#{OPTIONPARSERSUCKS}the inbox. Default: #{$opts[:archive]}.") { $opts[:archive] = true } + + opts.on("--scan-num N", Integer, "Number of messages to scan per source.#{OPTIONPARSERSUCKS}Default: #{$opts[:scan_num]}.") do |n| + $opts[:scan_num] = n + end + + opts.on_tail("-h", "--help", "Show this message") do + puts opts + exit + end +end.parse(ARGV) + +require "sup" +puts "loading index..." +index = Redwood::Index.new +index.load +puts "loaded index of #{index.size} messages" + +ARGV.each do |fn| + next if index.source_for fn + + ## TODO: merge this code with the same snippet in import + source = + case fn + when %r!^imaps?://! + print "Username for #{fn}: " + username = $stdin.gets.chomp + print "Password for #{fn} (warning: cleartext): " + password = $stdin.gets.chomp + Redwood::IMAP.new(fn, username, password, nil, !$opts[:unusual], $opts[:archive]) + else + Redwood::MBox::Loader.new(fn, nil, !$opts[:unusual], $opts[:archive]) + end + + source_ids = {} + count = 0 + source.each do |offset, labels| + begin + m = Redwood::Message.new :source => source, :source_info => offset + docid, entry = index.load_entry_for_id m.id + next unless entry + #puts "# #{source} #{offset} #{entry[:source_id]}" + + source_ids[entry[:source_id]] = (source_ids[entry[:source_id]] || 0) + 1 + count += 1 + break if count == $opts[:scan_num] + rescue Redwood::MessageFormatError => e + puts "# #{e.message}" + end + end + + if source_ids.size == 1 + id = source_ids.keys.first.to_i + puts "assigned #{source} to #{source_ids.keys.first}" + source.id = id + source.seek_to! source.total + index.add_source source + else + puts ">> unable to determine #{source}: #{source_ids.inspect}" + end +end + +index.save diff --git a/lib/sup.rb b/lib/sup.rb index 819fae1..703b396 100644 --- a/lib/sup.rb +++ b/lib/sup.rb @@ -28,7 +28,6 @@ module Redwood YAML_DATE = "2006-10-01" ## one-stop shop for yamliciousness - def register_yaml klass, props vars = props.map { |p| "@#{p}" } path = klass.name.gsub(/::/, "/") @@ -65,7 +64,6 @@ module Redwood end ## set up default configuration file - if File.exists? Redwood::CONFIG_FN $config = Redwood::load_yaml_obj Redwood::CONFIG_FN else @@ -93,6 +91,7 @@ require "sup/util" require "sup/update" require "sup/message" require "sup/mbox" +require "sup/imap" require "sup/person" require "sup/account" require "sup/thread" diff --git a/lib/sup/draft.rb b/lib/sup/draft.rb index 9847bae..5f8cce7 100644 --- a/lib/sup/draft.rb +++ b/lib/sup/draft.rb @@ -20,7 +20,7 @@ class DraftManager File.open(fn, "w") { |f| yield f } @source.each do |offset, labels| - m = Message.new @source, offset, labels + m = Message.new :source => @source, :source_info => offset, :labels => labels Index.add_message m UpdateManager.relay :add, m end diff --git a/lib/sup/imap.rb b/lib/sup/imap.rb new file mode 100644 index 0000000..a4559c3 --- /dev/null +++ b/lib/sup/imap.rb @@ -0,0 +1,91 @@ +require 'uri' +require 'net/imap' +require 'stringio' + +module Redwood + +class IMAP + attr_reader :uri + bool_reader :usual, :archived, :read, :dirty + attr_accessor :id, :labels + + class Error < StandardError; end + + def initialize uri, username, password, last_uid=nil, usual=true, archived=false, id=nil + raise "username and password must be specified" unless username && password + + @uri_s = uri + @uri = URI(uri) + @username = username + @password = password + @last_uid = last_uid || 1 + @dirty = false + @usual = usual + @archived = archived + @id = id + @imap = nil + @labels = [:unread, + archived ? nil : :inbox, + mailbox !~ /inbox/i && !mailbox.empty? ? mailbox.intern : nil, + ].compact + end + + def connect + return if @imap + Redwood::log "connecting to #{@uri.host} port #{ssl? ? 993 : 143}, ssl=#{ssl?}" + #raise "simulated imap failure" + @imap = Net::IMAP.new @uri.host, ssl? ? 993 : 143, ssl? + @imap.authenticate('LOGIN', @username, @password) + Redwood::log "success. selecting #{mailbox.inspect}." + @imap.examine(mailbox) + end + private :connect + + def mailbox; @uri.path[1..-1] end ##XXXX TODO handle nil + def ssl?; @uri.scheme == 'imaps' end + def reset!; @last_uid = 1; @dirty = true; end + def == o; o.is_a?(IMAP) && o.uri == uri; end + def uri; @uri.to_s; end + def to_s; uri; end + def is_source_for? s; to_s == s; end + + def load_header uid=nil + MBox::read_header StringIO.new(raw_header(uid)) + end + + def load_message uid + RMail::Parser.read raw_full_message(uid) + end + + ## load the full header text + def raw_header uid + connect + @imap.uid_fetch(uid, 'RFC822.HEADER')[0].attr['RFC822.HEADER'].gsub(/\r\n/, "\n") + end + + def raw_full_message uid + connect + @imap.uid_fetch(uid, 'RFC822')[0].attr['RFC822'].gsub(/\r\n/, "\n") + end + + def each + connect + uids = @imap.uid_search ['UID', "#{@last_uid}:#{total}"] + uids.each do |uid| + yield uid, labels + @last_uid = uid + @dirty = true + end + end + + def done?; @last_uid >= total; end + + def total + connect + @imap.uid_search(['ALL']).last + end +end + +Redwood::register_yaml(IMAP, %w(uri_s username password last_uid usual archived id)) + +end diff --git a/lib/sup/index.rb b/lib/sup/index.rb index 52d099e..6b41dd1 100644 --- a/lib/sup/index.rb +++ b/lib/sup/index.rb @@ -44,7 +44,8 @@ class Index raise "duplicate source!" if @sources.include? source @sources_dirty = true source.id ||= @sources.size - source.id += 1 while @sources.member? source.id + ##TODO: why was this necessary? + ##source.id += 1 while @sources.member? source.id @sources[source.id] = source end @@ -187,12 +188,12 @@ class Index raise "no snippet" unless doc[:snippet] begin - Message.new source, doc[:source_info].to_i, - doc[:label].split(" ").map { |s| s.intern }, - doc[:snippet] + Message.new :source => source, :source_info => doc[:source_info].to_i, + :labels => doc[:label].split(" ").map { |s| s.intern }, + :snippet => doc[:snippet] rescue MessageFormatError => e raise IndexError.new(source, "error building message #{doc[:message_id]} at #{source}/#{doc[:source_info]}: #{e.message}") -# rescue StandardError => e +# rescue StandardError => e # Message.new_from_index doc, < true if File.exists? fn + bakfn = fn + ".bak" + if File.exists? fn + File.chmod 0600, fn + FileUtils.mv fn, bakfn, :force => true unless File.exists?(bakfn) && File.size(bakfn) > File.size(fn) + end Redwood::save_yaml_obj @sources.values, fn + File.chmod 0600, fn end @sources_dirty = false end diff --git a/lib/sup/mbox/loader.rb b/lib/sup/mbox/loader.rb index dfc0dd2..5d26a30 100644 --- a/lib/sup/mbox/loader.rb +++ b/lib/sup/mbox/loader.rb @@ -46,12 +46,13 @@ class Loader @filename == s || self.to_s == s end - def load_header offset=nil + def load_header offset + raise ArgumentError, "nil offset" unless offset header = nil @mutex.synchronize do @f.seek offset if offset l = @f.gets - raise Error, "offset mismatch in mbox file: #{l.inspect}. Run 'sup-import --rebuild #{to_s}' to correct this." unless l =~ BREAK_RE + raise Error, "offset mismatch in mbox file offset #{offset.inspect}: #{l.inspect}. Run 'sup-import --rebuild #{to_s}' to correct this." unless l =~ BREAK_RE header = MBox::read_header @f end header diff --git a/lib/sup/message.rb b/lib/sup/message.rb index 9afde3f..d9614ba 100644 --- a/lib/sup/message.rb +++ b/lib/sup/message.rb @@ -76,25 +76,32 @@ class Message BLOCK_QUOTE_PATTERN = /^-----\s*Original Message\s*----+$/ QUOTE_START_PATTERN = /(^\s*Excerpts from)|(^\s*In message )|(^\s*In article )|(^\s*Quoting )|((wrote|writes|said|says)\s*:\s*$)/ SIG_PATTERN = /(^-- ?$)|(^\s*----------+\s*$)|(^\s*_________+\s*$)/ - SIG_DISTANCE = 15 # lines from the end + MAX_SIG_DISTANCE = 15 # lines from the end DEFAULT_SUBJECT = "(missing subject)" DEFAULT_SENDER = "(missing sender)" attr_reader :id, :date, :from, :subj, :refs, :replytos, :to, :source, :cc, :bcc, :labels, :list_address, :recipient_email, :replyto, - :source_info, :mbox_status + :source_info, :status bool_reader :dirty - def initialize source, source_info, labels, snippet=nil - @source = source - @source_info = source_info + ## if index_entry is specified, will fill in values from that, + def initialize opts + @source = opts[:source] + @source_info = opts[:source_info] + @snippet = opts[:snippet] || "" + @labels = opts[:labels] || [] @dirty = false - @snippet = snippet - @labels = labels - header = @source.load_header @source_info - header.each { |k, v| header[k.downcase] = v } + header = + if opts[:header] + opts[:header] + else + header = @source.load_header @source_info + header.each { |k, v| header[k.downcase] = v } + header + end %w(message-id date).each do |f| raise MessageFormatError, "no #{f} field in header #{header.inspect} (source #@source offset #@source_info)" unless header.include? f @@ -128,14 +135,10 @@ class Message end @recipient_email = header["delivered-to"] - @mbox_status = header["status"] - end - - def snippet - to_chunks unless @snippet - @snippet + @status = header["status"] end + def snippet; @snippet || to_chunks && @snippet; end def is_list_message?; !@list_address.nil?; end def is_draft?; DraftLoader === @source; end def draft_filename @@ -244,7 +247,7 @@ private newstate = nil if line =~ QUOTE_PATTERN || (line =~ QUOTE_START_PATTERN && (nextline =~ QUOTE_PATTERN || nextline =~ QUOTE_START_PATTERN)) newstate = :quote - elsif line =~ SIG_PATTERN && (lines.length - i) < SIG_DISTANCE + elsif line =~ SIG_PATTERN && (lines.length - i) < MAX_SIG_DISTANCE newstate = :sig elsif line =~ BLOCK_QUOTE_PATTERN newstate = :block_quote @@ -260,7 +263,7 @@ private newstate = nil if line =~ QUOTE_PATTERN || line =~ QUOTE_START_PATTERN || line =~ /^\s*$/ chunk_lines << line - elsif line =~ SIG_PATTERN && (lines.length - i) < SIG_DISTANCE + elsif line =~ SIG_PATTERN && (lines.length - i) < MAX_SIG_DISTANCE newstate = :sig else newstate = :text diff --git a/lib/sup/modes/resume-mode.rb b/lib/sup/modes/resume-mode.rb index b8b06b2..a1e273c 100644 --- a/lib/sup/modes/resume-mode.rb +++ b/lib/sup/modes/resume-mode.rb @@ -8,10 +8,26 @@ class ResumeMode < ComposeMode @header.delete "Date" @header["Message-Id"] = gen_message_id # generate a new'n regen_text + @sent = false end def send_message - DraftManager.discard @id if super + if super + DraftManager.discard @id + @sent = true + end + end + + def cleanup + unless @sent + if BufferManager.ask_yes_or_no "discard draft?" + DraftManager.discard @id + BufferManager.flash "Draft discarded." + else + BufferManager.flash "Draft saved." + end + super + end end end diff --git a/lib/sup/modes/thread-index-mode.rb b/lib/sup/modes/thread-index-mode.rb index d0cad36..923347b 100644 --- a/lib/sup/modes/thread-index-mode.rb +++ b/lib/sup/modes/thread-index-mode.rb @@ -180,8 +180,10 @@ class ThreadIndexMode < LineCursorMode threads = @threads + @hidden_threads.keys mbid = BufferManager.say "Saving threads..." threads.each_with_index do |t, i| - BufferManager.say "Saving thread #{i + 1} of #{threads.length}...", - mbid + if i % 5 == 0 + BufferManager.say "Saving thread #{i + 1} of #{threads.length}...", + mbid + end t.save Index end BufferManager.clear mbid diff --git a/lib/sup/poll.rb b/lib/sup/poll.rb index 8c7b793..2afe66e 100644 --- a/lib/sup/poll.rb +++ b/lib/sup/poll.rb @@ -57,9 +57,10 @@ class PollManager start_offset ||= offset yield " Found message at #{offset} with labels #{labels * ', '}" begin - m = Redwood::Message.new source, offset, labels + m = Redwood::Message.new :source => source, :source_info => offset, + :labels => labels if found[m.id] - yield "Skipping duplicate message #{m.id}" + yield "Skipping duplicate message #{m.id} (source total #{source.total})" next else found[m.id] = true diff --git a/lib/sup/sent.rb b/lib/sup/sent.rb index 44c1200..1da9adf 100644 --- a/lib/sup/sent.rb +++ b/lib/sup/sent.rb @@ -17,15 +17,12 @@ class SentManager def write_sent_message date, from_email need_blank = File.exists?(@fn) && !File.zero?(@fn) File.open(@fn, "a") do |f| - if need_blank - @source.increment_offset if @source.offset == f.tell - f.puts - end + f.puts if need_blank f.puts "From #{from_email} #{date}" yield f end @source.each do |offset, labels| - m = Message.new @source, offset, labels + m = Message.new :source => @source, :source_info => offset, :labels => labels Index.add_message m UpdateManager.relay :add, m end @@ -38,8 +35,6 @@ class SentLoader < MBox::Loader super filename, end_offset, true, true end - def increment_offset; @end_offset += 1; end - def offset; @end_offset; end def id; SentManager.source_id; end def to_s; SentManager.source_name; end