sup-sync [options] <source>*
where <source>* is zero or more source URIs. If no sources are given,
-sync from all usual sources.
-
-Supported source URIs:
- mbox://<path to mbox file>, e.g. mbox:///var/spool/mail/me
- maildir://<path to maildir dir>, e.g. maildir:///home/me/Maildir
- mbox+ssh://<machine>/<path to mbox file>
- imap://<machine>/[<folder>]
- imaps://<machine>/[<folder>]
+sync from all usual sources. Supported source URI schemes can be seen
+by running "sup-add --help".
Options controlling WHICH messages sup-sync operates on:
EOS
opt :new, "Operate on new messages only. Don't scan over the entire source. (Default.)", :short => :none
opt :changed, "Scan over the entire source for messages that have been deleted, altered, or moved from another source. (In the case of mbox sources, this includes all messages AFTER an altered message.)"
- opt :restored, "Operate only on those messages included in a dump file as specified by --restore."
+ opt :restored, "Operate only on those messages included in a dump file as specified by --restore which have changed state."
opt :all, "Operate on all messages in the source, regardless of newness or changedness."
opt :start_at, "For --changed and --all, start at a particular offset.", :type => :int
EOS
opt :verbose, "Print message ids as they're processed."
opt :optimize, "As the final operation, optimize the index."
+ opt :all_sources, "Scan over all sources.", :short => :none
opt :dry_run, "Don't actually modify the index. Probably only useful with --verbose.", :short => "-n"
opt :version, "Show version information", :short => :none
conflicts :changed, :all, :new, :restored
conflicts :asis, :restore, :discard
end
-Trollop::die :restored, "requires --restore" if opts[:restore] unless opts[:restored]
+Trollop::die :restored, "requires --restore" if opts[:restored] unless opts[:restore]
if opts[:start_at]
Trollop::die :start_at, "must be non-negative" if opts[:start_at] < 0
Trollop::die :start_at, "requires either --changed or --all" unless opts[:changed] || opts[:all]
Redwood::start
index = Redwood::Index.new
-index.load
restored_state =
if opts[:restore]
dump = {}
$stderr.puts "Loading state dump from #{opts[:restore]}..."
IO.foreach opts[:restore] do |l|
- l =~ /^(\S+) (\d+) (\d+) \((.*?)\)$/ or raise "Can't read dump line: #{l.inspect}"
- mid, source_id, source_info, labels = $1, $2.to_i, $3.to_i, $4
+ l =~ /^(\S+) \((.*?)\)$/ or raise "Can't read dump line: #{l.inspect}"
+ mid, labels = $1, $2
dump[mid] = labels.split(" ").map { |x| x.intern }
end
$stderr.puts "Read #{dump.size} entries from dump file."
{}
end
-sources = ARGV.map do |uri|
- uri = "mbox://#{uri}" unless uri =~ %r!://!
- index.source_for uri or Trollop::die "Unknown source: #{uri}. Did you add it with sup-add first?"
-end
-
-sources = index.usual_sources if sources.empty?
-
-unless target == :new
- if opts[:start_at]
- sources.each { |s| s.seek_to! opts[:start_at] }
- else
- sources.each { |s| s.reset! }
- end
-end
-
-last_info_time = start_time = Time.now
seen = {}
+index.lock_or_die
begin
+ index.load
+
+ sources = ARGV.map do |uri|
+ index.source_for uri or Trollop::die "Unknown source: #{uri}. Did you add it with sup-add first?"
+ end
+
+ sources = index.usual_sources if sources.empty?
+ sources = index.sources if opts[:all_sources]
+
+ unless target == :new
+ if opts[:start_at]
+ sources.each { |s| s.seek_to! opts[:start_at] }
+ else
+ sources.each { |s| s.reset! }
+ end
+ end
+
sources.each do |source|
- num_added, num_updated, num_scanned = 0, 0, 0
$stderr.puts "Scanning #{source}..."
+ num_added = num_updated = num_scanned = num_restored = 0
+ last_info_time = start_time = Time.now
Redwood::PollManager.add_messages_from source do |m, offset, entry|
num_scanned += 1
## reporting.
next if target == :changed && entry && entry[:source_id].to_i == source.id && entry[:source_info].to_i == offset
+ ## get the state currently in the index
+ index_state =
+ if entry
+ entry[:label].split(/\s+/).map { |x| x.intern }
+ else
+ nil
+ end
+
## skip if we're operating on restored messages, and this one
## ain't.
- next if target == :restored && !restored_state[m.id]
+ next if target == :restored && (!restored_state[m.id] || (index_state && restored_state[m.id].sort_by { |s| s.to_s } == index_state.sort_by { |s| s.to_s }))
## m.labels is the default source labels. tweak these according
## to default source state modification flags.
m.labels -= [:unread] if opts[:read]
m.labels += opts[:extra_labels].split(/\s*,\s*/).map { |x| x.intern } if opts[:extra_labels]
- ## get the state currently in the index
- index_state =
- if entry
- entry[:label].split(/\s+/).map { |x| x.intern }
- else
- nil
- end
-
## assign message labels based on the operation we're performing
case op
when :asis
## if the entry exists on disk
if restored_state[m.id]
m.labels = restored_state[m.id]
+ num_restored += 1
elsif index_state
m.labels = index_state
end
if Time.now - last_info_time > 60
last_info_time = Time.now
elapsed = last_info_time - start_time
- pctdone = source.respond_to?(:pct_done) ? source.pct_done : 100.0 * (source.cur_offset.to_f - source.start_time_offset).to_f / (source.end_offset - source.start_time_offset).to_f
+ pctdone = source.respond_to?(:pct_done) ? source.pct_done : 100.0 * (source.cur_offset.to_f - source.start_offset).to_f / (source.end_offset - source.start_offset).to_f
remaining = (100.0 - pctdone) * (elapsed.to_f / pctdone)
- puts "## #{num_added + num_updated} (#{pctdone}% done) read; #{elapsed.to_time_s} elapsed; est. #{remaining.to_time_s} remaining"
+ $stderr.puts "## #{num_scanned} (#{pctdone}%) read; #{elapsed.to_time_s} elapsed; #{remaining.to_time_s} remaining"
end
if index_state.nil?
opts[:dry_run] ? nil : m
end
- $stderr.puts "Added #{num_added}, updated #{num_updated} messages from #{source}."
+ $stderr.puts "Scanned #{num_scanned}, added #{num_added}, updated #{num_updated} messages from #{source}."
+ $stderr.puts "Restored state on #{num_restored} (#{100.0 * num_restored / num_scanned}%) messages." if num_restored > 0
end
-ensure
- $stderr.puts "Saving index and sources..."
- index.save
- Redwood::finish
-end
-## delete any messages in the index that claim they're from one of
-## these sources, but that we didn't see.
-##
-## kinda crappy code here, because we delve directly into the Ferret
-## API.
-##
-## TODO: move this to Index, i suppose.
-if target == :all || target == :changed
- $stderr.puts "Deleting missing messages from the index..."
- num_del, num_scanned = 0, 0
- sources.each do |source|
- raise "no source id for #{source}" unless source.id
- q = "+source_id:#{source.id}"
- q += " +source_info: >= #{opts[:start_at]}" if opts[:start_at]
- index.index.search_each(q, :limit => :all) do |docid, score|
- num_scanned += 1
- mid = index.index[docid][:message_id]
- unless seen[mid]
- puts "Deleting #{mid}" if opts[:verbose]
- index.index.delete docid unless opts[:dry_run]
- num_del += 1
+ ## delete any messages in the index that claim they're from one of
+ ## these sources, but that we didn't see.
+ ##
+ ## kinda crappy code here, because we delve directly into the Ferret
+ ## API.
+ ##
+ ## TODO: move this to Index, i suppose.
+
+
+ if target == :all || target == :changed
+ $stderr.puts "Deleting missing messages from the index..."
+ num_del, num_scanned = 0, 0
+ sources.each do |source|
+ raise "no source id for #{source}" unless source.id
+ q = "+source_id:#{source.id}"
+ q += " +source_info: >= #{opts[:start_at]}" if opts[:start_at]
+ index.index.search_each(q, :limit => :all) do |docid, score|
+ num_scanned += 1
+ mid = index.index[docid][:message_id]
+ unless seen[mid]
+ puts "Deleting #{mid}" if opts[:verbose]
+ index.index.delete docid unless opts[:dry_run]
+ num_del += 1
+ end
end
end
+ $stderr.puts "Deleted #{num_del} / #{num_scanned} messages"
end
- $stderr.puts "Deleted #{num_del} / #{num_scanned} messages"
-end
-if opts[:optimize]
- $stderr.puts "Optimizing index..."
- optt = time { index.index.optimize unless opts[:dry_run] }
- $stderr.puts "Optimized index of size #{index.size} in #{optt}s."
+ if opts[:optimize]
+ $stderr.puts "Optimizing index..."
+ optt = time { index.index.optimize unless opts[:dry_run] }
+ $stderr.puts "Optimized index of size #{index.size} in #{optt}s."
+ end
+rescue Redwood::FatalSourceError => e
+ $stderr.puts "Sorry, I couldn't communicate with a source: #{e.message}"
+rescue Exception => e
+ File.open("sup-exception-log.txt", "w") { |f| f.puts e.backtrace }
+ raise
+ensure
+ index.save
+ Redwood::finish
+ index.unlock
end
projects / sup / blobdiff