Merge branch 'sup-sync-improvements' into next

Conflicts:

	bin/sup-sync

diff --git a/bin/sup-sync b/bin/sup-sync
index 79343e6719b30875570444506feb6838dba3f6cd..01c0ebaac42d452760c1e20999f2e4c37a50f597 100644 (file)
--- a/bin/sup-sync
+++ b/bin/sup-sync
@@ -124,7 +124,9 @@ begin
 
   unless target == :new
     if opts[:start_at]
-      sources.each { |s| s.seek_to! opts[:start_at] }
+      Trollop::die :start_at, "can only be used on one source" unless sources.size == 1
+      sources.first.seek_to! opts[:start_at]
+      sources.first.correct_offset! if sources.first.respond_to?(:correct_offset!)
     else
       sources.each { |s| s.reset! }
     end
@@ -139,6 +141,15 @@ begin
       num_scanned += 1
       seen[m.id] = true
 
+      if Time.now - last_info_time > PROGRESS_UPDATE_INTERVAL
+        last_info_time = Time.now
+        elapsed = last_info_time - start_time
+        start = opts[:start_at] || source.start_offset
+        pctdone = 100.0 * (source.cur_offset - start).to_f / (source.end_offset - start).to_f
+        remaining = (100.0 - pctdone) * (elapsed.to_f / pctdone)
+        $stderr.printf "## read %dm (about %.0f%%) @ %.1fm/s. %s elapsed, about %s remaining\n", num_scanned, pctdone, num_scanned / elapsed, elapsed.to_time_s, remaining.to_time_s
+      end
+
       ## skip if we're operating only on changed messages, the message
       ## is in the index, and it's unchanged from what the source is
       ## reporting.
@@ -182,7 +193,7 @@ begin
       end
 
       if index_state.nil?
-        puts "Adding message #{source}##{offset} with state {#{m.labels * ', '}}" if opts[:verbose]
+        puts "Adding message #{source}##{offset} from #{m.from} with state {#{m.labels * ', '}}" if opts[:verbose]
         num_added += 1
       else
         puts "Updating message #{source}##{offset}, source #{entry[:source_id]} => #{source.id}, offset #{entry[:source_info]} => #{offset}, state {#{index_state * ', '}} => {#{m.labels * ', '}}" if opts[:verbose]
@@ -202,7 +213,7 @@ begin
   ## API.
   ##
   ## TODO: move this to Index, i suppose.
-  if target == :all || target == :changed
+  if (target == :all || target == :changed) && !opts[:start_at]
     $stderr.puts "Deleting missing messages from the index..."
     num_del, num_scanned = 0, 0
     sources.each do |source|

diff --git a/lib/sup/mbox/loader.rb b/lib/sup/mbox/loader.rb
index 3d8ef6aa2d7ae5d17881c0cee57a0eca85b7131c..c35d0c8cd05afe0d2b1de8b6db668d79f9ef1681 100644 (file)
--- a/lib/sup/mbox/loader.rb
+++ b/lib/sup/mbox/loader.rb
@@ -80,6 +80,18 @@ class Loader < Source
     end
   end
 
+  ## scan forward until we're at the valid start of a message
+  def correct_offset!
+    @mutex.synchronize do
+      @f.seek cur_offset
+      string = ""
+      until @f.eof? || (l = @f.gets) =~ BREAK_RE
+        string << l
+      end
+      self.cur_offset += string.length
+    end
+  end
+
   def raw_header offset
     ret = ""
     @mutex.synchronize do