From: wmorgan Date: Sun, 20 May 2007 23:43:15 +0000 (+0000) Subject: added sup-sync-back X-Git-Url: https://git.notmuchmail.org/git?a=commitdiff_plain;h=fea4150a5e4948d4f98ba2d88ba74ee459301d30;p=sup added sup-sync-back git-svn-id: svn://rubyforge.org/var/svn/sup/trunk@405 5c8cc53c-5e98-4d25-b20a-d8db53a31250 --- diff --git a/bin/sup-sync b/bin/sup-sync index 14d72f4..12a830a 100644 --- a/bin/sup-sync +++ b/bin/sup-sync @@ -43,8 +43,8 @@ Usage: sup-sync [options] * where * is zero or more source URIs. If no sources are given, -sync from all usual sources. All supported source URI schemes can -be seen by running "sup-add --help". +sync from all usual sources. Supported source URI schemes can be seen +by running "sup-add --help". Options controlling WHICH messages sup-sync operates on: EOS diff --git a/bin/sup-sync-back b/bin/sup-sync-back new file mode 100644 index 0000000..608a1ea --- /dev/null +++ b/bin/sup-sync-back @@ -0,0 +1,124 @@ +#!/usr/bin/env ruby + +require 'rubygems' +require 'uri' +require 'tempfile' +require 'trollop' +require "sup" + +## save a message 'm' to an open file pointer 'fp' +def save m, fp + m.source.each_raw_full_message_line(m.source_info) { |l| fp.print l } +end + +opts = Trollop::options do + version "sup-sync-back (sup #{Redwood::VERSION})" + banner <* + +where * is zero or more source URIs. If no sources are given, +sync back all usual sources. + +You probably want to run sup-sync --changed after this command. + +Options include: +EOS + opt :delete_deleted, "Delete deleted messages.", :default => false, :short => "d" + opt :move_deleted, "Move deleted messages to a local mbox file.", :type => String, :short => :none + opt :delete_spam, "Delete spam messages.", :default => false, :short => "s" + opt :move_spam, "Move spam messages to a local mbox file.", :type => String, :short => :none + opt :verbose, "Print message ids as they're processed." + opt :dry_run, "Don't actually modify the index. Probably only useful with --verbose.", :short => "-n" + opt :version, "Show version information", :short => :none + + conflicts :delete_deleted, :move_deleted + conflicts :delete_spam, :move_spam +end + +Redwood::start +index = Redwood::Index.new +index.load + +sources = ARGV.map do |uri| + s = index.source_for(uri) or Trollop::die "Unknown source: #{uri}. Did you add it with sup-add first?" + s.is_a?(Redwood::MBox::Loader) or Trollop::die "#{uri} is not an mbox source." + s +end + +unless opts[:dry_run] + deleted_fp = File.open(opts[:move_deleted], "a") if opts[:move_deleted] + spam_fp = File.open(opts[:move_spam], "a") if opts[:move_spam] +end + +begin + sources.each { |s| s.reset! } + + sources.each do |source| + $stderr.puts "Scanning #{source}..." + num_deleted = num_moved = num_scanned = 0 + + out_fp = Tempfile.new "sup-sync-back-#{source.id}" + Redwood::PollManager.add_messages_from source do |m, offset, entry| + num_scanned += 1 + + if entry + labels = entry[:label].split.map { |x| x.intern }.to_boolean_h + + if labels.member? :deleted + if opts[:delete_deleted] + puts "Dropping deleted message #{source}##{offset}" if opts[:verbose] + num_deleted += 1 + elsif opts[:move_deleted] && labels.member?(:deleted) + puts "Moving deleted message #{source}##{offset}" if opts[:verbose] + save m, deleted_fp unless opts[:dry_run] + num_moved += 1 + end + + elsif labels.member? :spam + if opts[:delete_spam] + puts "Deleting spam message #{source}##{offset}" if opts[:verbose] + num_deleted += 1 + elsif opts[:move_spam] && labels.member?(:spam) + puts "Moving spam message #{source}##{offset}" if opts[:verbose] + save m, spam_fp unless opts[:dry_run] + num_moved += 1 + end + else + save m, out_fp unless opts[:dry_run] + end + else + save m, out_fp unless opts[:dry_run] + end + + nil # don't actually add anything! + end + $stderr.puts "Scanned #{num_scanned}, deleted #{num_deleted}, moved #{num_moved} messages from #{source}." + out_fp.close unless opts[:dry_run] + + unless opts[:dry_run] + deleted_fp.flush + spam_fp.flush + out_fp.close + FileUtils.mv out_fp.path, URI(source.uri).path + end + end + + unless opts[:dry_run] + deleted_fp.close + spam_fp.close + end + + $stderr.puts "Done. You should run sup-sync --changed #{sources.join(' ')}" +rescue Exception => e + File.open("sup-exception-log.txt", "w") { |f| f.puts e.backtrace } + raise +ensure + Redwood::finish +end diff --git a/doc/TODO b/doc/TODO index cbb6c99..853abcf 100644 --- a/doc/TODO +++ b/doc/TODO @@ -11,6 +11,8 @@ _ bugfix: readlock _ split out threading & message chunk parsing to a separate library _ decode RFC 2047 ("encoded word") headers - see: http://blade.nagaokaut.ac.jp/cgi-bin/scat.rb/ruby/ruby-talk/101949, http://dev.rubyonrails.org/ticket/6807 +x add a sync-back tool that at least works for mboxes +x thread by subject configurable in config.yaml x view as text command if the mime view command fails for an attachment x bugfix: attachment filenames sometimes not detected (filename=) x bugfix: rmail multipart error diff --git a/lib/sup/mbox/loader.rb b/lib/sup/mbox/loader.rb index e6fd1d4..49eb37b 100644 --- a/lib/sup/mbox/loader.rb +++ b/lib/sup/mbox/loader.rb @@ -73,14 +73,24 @@ class Loader < Source def raw_full_message offset ret = "" + each_raw_full_message_line(offset) { |l| ret += l } + ret + end + + ## apparently it's a million times faster to call this directly if + ## we're just moving messages around on disk, than reading things + ## into memory with raw_full_message. + ## + ## i hoped never to have to move shit around on disk but + ## sup-sync-back has to do it. + def each_raw_full_message_line offset @mutex.synchronize do @f.seek offset - @f.gets # skip mbox header + yield @f.gets until @f.eof? || (l = @f.gets) =~ BREAK_RE - ret += l + yield l end end - ret end def next