unbroke

[sup] / bin / sup-import

diff --git a/bin/sup-import b/bin/sup-import
index 69b5b072515c8fc970e01a583e7648b9fec0afd0..f5de3c2ed6d2a73b2405cb3e0d8ff146ab6b6f4a 100644 (file)
--- a/bin/sup-import
+++ b/bin/sup-import
@@ -32,9 +32,9 @@ Imports messages into the Sup index from one or more sources.
 Usage:
   sup-import [options] <source>*
 
-where <source>* is zero or more source URIs or mbox filenames, e.g.
-"imaps://my.imapserver.com", or "/var/spool/mail/me". If no sources
-are given, imports messages from all sources marked as "usual".
+where <source>* is zero or more source URIs or mbox filenames. If no
+sources are given, imports messages from all sources marked as
+"usual".
 
 Options are:
 EOS
@@ -76,14 +76,14 @@ if opts[:rebuild] || opts[:full_rebuild]
   end
 end
 
-start = Time.now
+last_update = start = Time.now
 found = {}
 begin
   sources.each do |source|
-    num = 0
+    num_added = 0
+    num_updated = 0
+    puts "Scanning #{source}..."
     Redwood::PollManager.add_new_messages_from source do |m, offset, entry|
-      found[m.id] = true
-
       ## if the entry exists on disk
       if entry && !opts[:overwrite_state]
         m.labels = entry[:label].split(/\s+/).map { |x| x.intern }
@@ -93,31 +93,35 @@ begin
         m.labels -= [:unread] if opts[:read]
       end
 
-      if num % 1000 == 0 && num > 0
-        elapsed = Time.now - start
-        pctdone = source.pct_done
+      if Time.now - last_update > 60
+        last_update = Time.now
+        elapsed = last_update - start
+        pctdone = source.respond_to?(:pct_done) ? source.pct_done : 100.0 * (source.cur_offset.to_f - source.start_offset).to_f / (source.end_offset - source.start_offset).to_f
         remaining = (100.0 - pctdone) * (elapsed.to_f / pctdone)
         puts "## #{num} (#{pctdone}% done) read; #{elapsed.to_time_s} elapsed; est. #{remaining.to_time_s} remaining"
       end
 
       ## update if...
       if entry.nil? # it's a new message; or
-        puts "# adding message at #{offset}, labels: #{m.labels * ' '}" if opts[:verbose]
-        num += 1
+        puts "Adding message at #{offset}, labels: #{m.labels * ' '}" if opts[:verbose]
+        num_added += 1
+        found[m.id] = true
         m
       elsif opts[:full_rebuild] || # we're updating everyone; or
           (opts[:rebuild] && (entry[:source_id].to_i != source.id || entry[:source_info].to_i != offset)) # we're updating just the changed ones
-        puts "# updating message at #{offset}, source #{entry[:source_id]} => #{source.id}, offset #{entry[:source_info]} => #{offset}, labels: #{m.labels * ' '}" if opts[:verbose]
-        num += 1
+        puts "Updating message at #{offset} (from #{m.from.longname}, subject '#{m.subj}'), source #{entry[:source_id]} => #{source.id}, offset #{entry[:source_info]} => #{offset}, labels: {#{m.labels * ', '}}" if opts[:verbose]
+        num_updated += 1 unless found[m.id]
+        found[m.id] = true
         m
       else
+        found[m.id] = true
         nil
       end
     end
-    puts "loaded #{num} messages from #{source}" unless num == 0
+    puts "Added #{num_added}, updated #{num_updated} messages from #{source}."
   end
 ensure
-  $stderr.puts "saving index and sources..."
+  puts "Saving index and sources..."
   index.save
   Redwood::finish
 end
@@ -130,7 +134,7 @@ end
 ##
 ## TODO: move this to Index, i suppose.
 if opts[:rebuild] || opts[:full_rebuild]
-  puts "deleting missing messages from the index..."
+  puts "Deleting missing messages from the index..."
   numdel = num = 0
   sources.each do |source|
     raise "no source id for #{source}" unless source.id
@@ -140,16 +144,16 @@ if opts[:rebuild] || opts[:full_rebuild]
       mid = index.index[docid][:message_id]
 #      puts "got #{mid}"
       next if found[mid]
-      puts "deleting #{mid}"
+      puts "Deleting #{mid}" if opts[:verbose]
       index.index.delete docid
       numdel += 1
     end
   end
-  puts "deleted #{numdel} / #{num} messages"
+  puts "Deleted #{numdel} / #{num} messages"
 end
 
 if opts[:optimize]
-  puts "optimizing index..."
+  puts "Optimizing index..."
   optt = time { index.index.optimize }
-  puts "optimized index of size #{index.size} in #{optt}s."
+  puts "Optimized index of size #{index.size} in #{optt}s."
 end