9 def to_s; sprintf '%.2f', self; end
15 sprintf "%d:%02d:%02d", i / 3600, (i / 60) % 60, i % 60
25 opts = Trollop::options do
26 version "sup-sync (sup #{Redwood::VERSION})"
28 Synchronizes the Sup index with one or more message sources by adding
29 messages, deleting messages, or changing message state in the index as
32 "Message state" means read/unread, archived/inbox, starred/unstarred,
33 and all user-defined labels on each message.
35 "Default source state" refers to any state that a source itself has
36 keeps about a message. Sup-sync uses this information when adding a
37 new message to the index. The source state is typically limited to
38 read/unread, archived/inbox status and a single label based on the
39 source name. Messages using the default source state are placed in
40 the inbox (i.e. not archived) and unstarred.
43 sup-sync [options] <source>*
45 where <source>* is zero or more source URIs. If no sources are given,
46 sync from all usual sources. Supported source URI schemes can be seen
47 by running "sup-add --help".
49 Options controlling WHICH messages sup-sync operates on:
51 opt :new, "Operate on new messages only. Don't scan over the entire source. (Default.)", :short => :none
52 opt :changed, "Scan over the entire source for messages that have been deleted, altered, or moved from another source. (In the case of mbox sources, this includes all messages AFTER an altered message.)"
53 opt :restored, "Operate only on those messages included in a dump file as specified by --restore which have changed state."
54 opt :all, "Operate on all messages in the source, regardless of newness or changedness."
55 opt :start_at, "For --changed and --all, start at a particular offset.", :type => :int
59 Options controlling HOW message state is altered:
61 opt :asis, "If the message is already in the index, preserve its state. Otherwise, use default source state. (Default.)", :short => :none
62 opt :restore, "Restore message state from a dump file created with sup-dump. If a message is not in this dumpfile, act as --asis.", :type => String, :short => :none
63 opt :discard, "Discard any message state in the index and use the default source state. Dangerous!", :short => :none
64 opt :archive, "When using the default source state, mark messages as archived.", :short => "-x"
65 opt :read, "When using the default source state, mark messages as read."
66 opt :extra_labels, "When using the default source state, also apply these user-defined labels. Should be a comma-separated list.", :type => String, :short => :none
72 opt :verbose, "Print message ids as they're processed."
73 opt :optimize, "As the final operation, optimize the index."
74 opt :all_sources, "Scan over all sources.", :short => :none
75 opt :dry_run, "Don't actually modify the index. Probably only useful with --verbose.", :short => "-n"
76 opt :version, "Show version information", :short => :none
78 conflicts :changed, :all, :new, :restored
79 conflicts :asis, :restore, :discard
81 Trollop::die :restored, "requires --restore" if opts[:restored] unless opts[:restore]
83 Trollop::die :start_at, "must be non-negative" if opts[:start_at] < 0
84 Trollop::die :start_at, "requires either --changed or --all" unless opts[:changed] || opts[:all]
87 target = [:new, :changed, :all, :restored].find { |x| opts[x] } || :new
88 op = [:asis, :restore, :discard].find { |x| opts[x] } || :asis
91 index = Redwood::Index.new
97 $stderr.puts "Loading state dump from #{opts[:restore]}..."
98 IO.foreach opts[:restore] do |l|
99 l =~ /^(\S+) \((.*?)\)$/ or raise "Can't read dump line: #{l.inspect}"
101 dump[mid] = labels.split(" ").map { |x| x.intern }
103 $stderr.puts "Read #{dump.size} entries from dump file."
109 sources = ARGV.map do |uri|
110 index.source_for uri or Trollop::die "Unknown source: #{uri}. Did you add it with sup-add first?"
113 sources = index.usual_sources if sources.empty?
114 sources = index.sources if opts[:all_sources]
118 unless target == :new
120 sources.each { |s| s.seek_to! opts[:start_at] }
122 sources.each { |s| s.reset! }
126 sources.each do |source|
127 $stderr.puts "Scanning #{source}..."
128 num_added = num_updated = num_scanned = num_restored = 0
129 last_info_time = start_time = Time.now
131 Redwood::PollManager.add_messages_from source do |m, offset, entry|
135 ## skip if we're operating only on changed messages, the message
136 ## is in the index, and it's unchanged from what the source is
138 next if target == :changed && entry && entry[:source_id].to_i == source.id && entry[:source_info].to_i == offset
140 ## get the state currently in the index
143 entry[:label].split(/\s+/).map { |x| x.intern }
148 ## skip if we're operating on restored messages, and this one
150 next if target == :restored && (!restored_state[m.id] || restored_state[m.id].sort_by { |s| s.to_s } == index_state.sort_by { |s| s.to_s })
152 ## m.labels is the default source labels. tweak these according
153 ## to default source state modification flags.
154 m.labels -= [:inbox] if opts[:archive]
155 m.labels -= [:unread] if opts[:read]
156 m.labels += opts[:extra_labels].split(/\s*,\s*/).map { |x| x.intern } if opts[:extra_labels]
158 ## assign message labels based on the operation we're performing
161 m.labels = index_state if index_state
163 ## if the entry exists on disk
164 if restored_state[m.id]
165 m.labels = restored_state[m.id]
168 m.labels = index_state
171 ## nothin! use default source labels
174 if Time.now - last_info_time > 60
175 last_info_time = Time.now
176 elapsed = last_info_time - start_time
177 pctdone = source.respond_to?(:pct_done) ? source.pct_done : 100.0 * (source.cur_offset.to_f - source.start_offset).to_f / (source.end_offset - source.start_offset).to_f
178 remaining = (100.0 - pctdone) * (elapsed.to_f / pctdone)
179 $stderr.puts "## #{num_added + num_updated} (#{pctdone}%) read; #{elapsed.to_time_s} elapsed; #{remaining.to_time_s} remaining"
183 puts "Adding message #{source}##{offset} with state {#{m.labels * ', '}}" if opts[:verbose]
186 puts "Updating message #{source}##{offset}, source #{entry[:source_id]} => #{source.id}, offset #{entry[:source_info]} => #{offset}, state {#{index_state * ', '}} => {#{m.labels * ', '}}" if opts[:verbose]
190 opts[:dry_run] ? nil : m
192 $stderr.puts "Scanned #{num_scanned}, added #{num_added}, updated #{num_updated} messages from #{source}."
193 $stderr.puts "Restored state on #{num_restored} (#{100.0 * num_restored / num_scanned}%) messages." if num_restored > 0
195 rescue Redwood::FatalSourceError => e
196 $stderr.puts "Sorry, I couldn't communicate with a source: #{e.message}"
197 rescue Exception => e
198 File.open("sup-exception-log.txt", "w") { |f| f.puts e.backtrace }
205 ## delete any messages in the index that claim they're from one of
206 ## these sources, but that we didn't see.
208 ## kinda crappy code here, because we delve directly into the Ferret
211 ## TODO: move this to Index, i suppose.
212 if target == :all || target == :changed
213 $stderr.puts "Deleting missing messages from the index..."
214 num_del, num_scanned = 0, 0
215 sources.each do |source|
216 raise "no source id for #{source}" unless source.id
217 q = "+source_id:#{source.id}"
218 q += " +source_info: >= #{opts[:start_at]}" if opts[:start_at]
219 index.index.search_each(q, :limit => :all) do |docid, score|
221 mid = index.index[docid][:message_id]
223 puts "Deleting #{mid}" if opts[:verbose]
224 index.index.delete docid unless opts[:dry_run]
229 $stderr.puts "Deleted #{num_del} / #{num_scanned} messages"
233 $stderr.puts "Optimizing index..."
234 optt = time { index.index.optimize unless opts[:dry_run] }
235 $stderr.puts "Optimized index of size #{index.size} in #{optt}s."