3 # Copyright (c) 2011-2014 David Bremner <david@tethera.net>
4 # W. Trevor King <wking@tremily.us>
6 # This program is free software: you can redistribute it and/or modify
7 # it under the terms of the GNU General Public License as published by
8 # the Free Software Foundation, either version 3 of the License, or
9 # (at your option) any later version.
11 # This program is distributed in the hope that it will be useful,
12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 # GNU General Public License for more details.
16 # You should have received a copy of the GNU General Public License
17 # along with this program. If not, see https://www.gnu.org/licenses/ .
20 Manage notmuch tags with Git
22 Environment variables:
24 * NMBGIT specifies the location of the git repository used by nmbug.
25 If not specified $HOME/.nmbug is used.
26 * NMBPREFIX specifies the prefix in the notmuch database for tags of
27 interest to nmbug. If not specified 'notmuch::' is used.
30 from __future__ import print_function
31 from __future__ import unicode_literals
33 import codecs as _codecs
34 import collections as _collections
35 import functools as _functools
36 import inspect as _inspect
37 import locale as _locale
38 import logging as _logging
41 import shutil as _shutil
42 import subprocess as _subprocess
44 import tempfile as _tempfile
45 import textwrap as _textwrap
47 from urllib.parse import quote as _quote
48 from urllib.parse import unquote as _unquote
49 except ImportError: # Python 2
50 from urllib import quote as _quote
51 from urllib import unquote as _unquote
56 _LOG = _logging.getLogger('nmbug')
57 _LOG.setLevel(_logging.WARNING)
58 _LOG.addHandler(_logging.StreamHandler())
60 NMBGIT = _os.path.expanduser(
61 _os.getenv('NMBGIT', _os.path.join('~', '.nmbug')))
62 _NMBGIT = _os.path.join(NMBGIT, '.git')
63 if _os.path.isdir(_NMBGIT):
66 TAG_PREFIX = _os.getenv('NMBPREFIX', 'notmuch::')
67 _HEX_ESCAPE_REGEX = _re.compile('%[0-9A-F]{2}')
68 _TAG_FILE_REGEX = _re.compile('tags/(?P<id>[^/]*)/(?P<tag>[^/]*)')
70 # magic hash for Git (git hash-object -t blob /dev/null)
71 _EMPTYBLOB = 'e69de29bb2d1d6434b8b29ae775ad8c2e48c5391'
75 getattr(_tempfile, 'TemporaryDirectory')
76 except AttributeError: # Python < 3.2
77 class _TemporaryDirectory(object):
79 Fallback context manager for Python < 3.2
81 See PEP 343 for details on context managers [1].
83 [1]: https://www.python.org/dev/peps/pep-0343/
85 def __init__(self, **kwargs):
86 self.name = _tempfile.mkdtemp(**kwargs)
91 def __exit__(self, type, value, traceback):
92 _shutil.rmtree(self.name)
95 _tempfile.TemporaryDirectory = _TemporaryDirectory
98 def _hex_quote(string, safe='+@=:,'):
100 quote('abc def') -> 'abc%20def'.
102 Wrap urllib.parse.quote with additional safe characters (in
103 addition to letters, digits, and '_.-') and lowercase hex digits
104 (e.g. '%3a' instead of '%3A').
106 uppercase_escapes = _quote(string, safe)
107 return _HEX_ESCAPE_REGEX.sub(
108 lambda match: match.group(0).lower(),
112 _ENCODED_TAG_PREFIX = _hex_quote(TAG_PREFIX, safe='+@=,') # quote ':'
115 def _xapian_quote(string):
117 Quote a string for Xapian's QueryParser.
119 Xapian uses double-quotes for quoting strings. You can escape
120 internal quotes by repeating them [1,2,3].
122 [1]: https://trac.xapian.org/ticket/128#comment:2
123 [2]: https://trac.xapian.org/ticket/128#comment:17
124 [3]: https://trac.xapian.org/changeset/13823/svn
126 return '"{0}"'.format(string.replace('"', '""'))
129 def _xapian_unquote(string):
131 Unquote a Xapian-quoted string.
133 if string.startswith('"') and string.endswith('"'):
134 return string[1:-1].replace('""', '"')
138 class SubprocessError(RuntimeError):
139 "A subprocess exited with a nonzero status"
140 def __init__(self, args, status, stdout=None, stderr=None):
144 msg = '{args} exited with {status}'.format(args=args, status=status)
146 msg = '{msg}: {stderr}'.format(msg=msg, stderr=stderr)
147 super(SubprocessError, self).__init__(msg)
150 class _SubprocessContextManager(object):
152 PEP 343 context manager for subprocesses.
154 'expect' holds a tuple of acceptable exit codes, otherwise we'll
155 raise a SubprocessError in __exit__.
157 def __init__(self, process, args, expect=(0,)):
158 self._process = process
160 self._expect = expect
165 def __exit__(self, type, value, traceback):
166 for name in ['stdin', 'stdout', 'stderr']:
167 stream = getattr(self._process, name)
170 setattr(self._process, name, None)
171 status = self._process.wait()
173 'collect {args} with status {status} (expected {expect})'.format(
174 args=self._args, status=status, expect=self._expect))
175 if status not in self._expect:
176 raise SubprocessError(args=self._args, status=status)
179 return self._process.wait()
182 def _spawn(args, input=None, additional_env=None, wait=False, stdin=None,
183 stdout=None, stderr=None, encoding=_locale.getpreferredencoding(),
184 expect=(0,), **kwargs):
185 """Spawn a subprocess, and optionally wait for it to finish.
187 This wrapper around subprocess.Popen has two modes, depending on
188 the truthiness of 'wait'. If 'wait' is true, we use p.communicate
189 internally to write 'input' to the subprocess's stdin and read
190 from it's stdout/stderr. If 'wait' is False, we return a
191 _SubprocessContextManager instance for fancier handling
192 (e.g. piping between processes).
194 For 'wait' calls when you want to write to the subprocess's stdin,
195 you only need to set 'input' to your content. When 'input' is not
196 None but 'stdin' is, we'll automatically set 'stdin' to PIPE
197 before calling Popen. This avoids having the subprocess
198 accidentally inherit the launching process's stdin.
200 _LOG.debug('spawn {args} (additional env. var.: {env})'.format(
201 args=args, env=additional_env))
202 if not stdin and input is not None:
203 stdin = _subprocess.PIPE
205 if not kwargs.get('env'):
206 kwargs['env'] = dict(_os.environ)
207 kwargs['env'].update(additional_env)
208 p = _subprocess.Popen(
209 args, stdin=stdin, stdout=stdout, stderr=stderr, **kwargs)
211 if hasattr(input, 'encode'):
212 input = input.encode(encoding)
213 (stdout, stderr) = p.communicate(input=input)
216 'collect {args} with status {status} (expected {expect})'.format(
217 args=args, status=status, expect=expect))
218 if stdout is not None:
219 stdout = stdout.decode(encoding)
220 if stderr is not None:
221 stderr = stderr.decode(encoding)
222 if status not in expect:
223 raise SubprocessError(
224 args=args, status=status, stdout=stdout, stderr=stderr)
225 return (status, stdout, stderr)
226 if p.stdin and not stdin:
230 p.stdin = _codecs.getwriter(encoding=encoding)(stream=p.stdin)
231 stream_reader = _codecs.getreader(encoding=encoding)
233 p.stdout = stream_reader(stream=p.stdout)
235 p.stderr = stream_reader(stream=p.stderr)
236 return _SubprocessContextManager(args=args, process=p, expect=expect)
239 def _git(args, **kwargs):
240 args = ['git', '--git-dir', NMBGIT] + list(args)
241 return _spawn(args=args, **kwargs)
244 def _get_current_branch():
245 """Get the name of the current branch.
247 Return 'None' if we're not on a branch.
250 (status, branch, stderr) = _git(
251 args=['symbolic-ref', '--short', 'HEAD'],
252 stdout=_subprocess.PIPE, stderr=_subprocess.PIPE, wait=True)
253 except SubprocessError as e:
254 if 'not a symbolic ref' in e:
257 return branch.strip()
261 "Get the default remote for the current branch."
262 local_branch = _get_current_branch()
263 (status, remote, stderr) = _git(
264 args=['config', 'branch.{0}.remote'.format(local_branch)],
265 stdout=_subprocess.PIPE, wait=True)
266 return remote.strip()
269 def get_tags(prefix=None):
270 "Get a list of tags with a given prefix."
273 (status, stdout, stderr) = _spawn(
274 args=['notmuch', 'search', '--output=tags', '*'],
275 stdout=_subprocess.PIPE, wait=True)
276 return [tag for tag in stdout.splitlines() if tag.startswith(prefix)]
279 def archive(treeish='HEAD', args=()):
281 Dump a tar archive of the current nmbug tag set.
285 Each tag $tag for message with Message-Id $id is written to
288 tags/encode($id)/encode($tag)
290 The encoding preserves alphanumerics, and the characters
291 "+-_@=.:," (not the quotes). All other octets are replaced with
292 '%' followed by a two digit hex number.
294 _git(args=['archive', treeish] + list(args), wait=True)
297 def clone(repository):
299 Create a local nmbug repository from a remote source.
301 This wraps 'git clone', adding some options to avoid creating a
302 working tree while preserving remote-tracking branches and
305 with _tempfile.TemporaryDirectory(prefix='nmbug-clone.') as workdir:
308 'git', 'clone', '--no-checkout', '--separate-git-dir', NMBGIT,
309 repository, workdir],
311 _git(args=['config', '--unset', 'core.worktree'], wait=True, expect=(0, 5))
312 _git(args=['config', 'core.bare', 'true'], wait=True)
313 _git(args=['branch', 'config', 'origin/config'], wait=True)
314 existing_tags = get_tags()
317 'Not checking out to avoid clobbering existing tags: {}'.format(
318 ', '.join(existing_tags)))
323 def _is_committed(status):
324 return len(status['added']) + len(status['deleted']) == 0
327 def commit(treeish='HEAD', message=None):
329 Commit prefix-matching tags from the notmuch database to Git.
331 status = get_status()
333 if _is_committed(status=status):
334 _LOG.warning('Nothing to commit')
337 _git(args=['read-tree', '--empty'], wait=True)
338 _git(args=['read-tree', treeish], wait=True)
340 _update_index(status=status)
343 stdout=_subprocess.PIPE,
345 (_, parent, _) = _git(
346 args=['rev-parse', treeish],
347 stdout=_subprocess.PIPE,
349 (_, commit, _) = _git(
350 args=['commit-tree', tree.strip(), '-p', parent.strip()],
352 stdout=_subprocess.PIPE,
355 args=['update-ref', treeish, commit.strip()],
356 stdout=_subprocess.PIPE,
358 except Exception as e:
359 _git(args=['read-tree', '--empty'], wait=True)
360 _git(args=['read-tree', treeish], wait=True)
363 def _update_index(status):
365 args=['update-index', '--index-info'],
366 stdin=_subprocess.PIPE) as p:
367 for id, tags in status['deleted'].items():
368 for line in _index_tags_for_message(id=id, status='D', tags=tags):
370 for id, tags in status['added'].items():
371 for line in _index_tags_for_message(id=id, status='A', tags=tags):
375 def fetch(remote=None):
377 Fetch changes from the remote repository.
379 See 'merge' to bring those changes into notmuch.
384 _git(args=args, wait=True)
389 Update the notmuch database from Git.
391 This is mainly useful to discard your changes in notmuch relative
394 status = get_status()
396 args=['notmuch', 'tag', '--batch'], stdin=_subprocess.PIPE) as p:
397 for id, tags in status['added'].items():
398 p.stdin.write(_batch_line(action='-', id=id, tags=tags))
399 for id, tags in status['deleted'].items():
400 p.stdin.write(_batch_line(action='+', id=id, tags=tags))
403 def _batch_line(action, id, tags):
405 'notmuch tag --batch' line for adding/removing tags.
407 Set 'action' to '-' to remove a tag or '+' to add the tags to a
410 tag_string = ' '.join(
411 '{action}{prefix}{tag}'.format(
412 action=action, prefix=_ENCODED_TAG_PREFIX, tag=_hex_quote(tag))
414 line = '{tags} -- id:{id}\n'.format(
415 tags=tag_string, id=_xapian_quote(string=id))
419 def _insist_committed():
420 "Die if the the notmuch tags don't match the current HEAD."
421 status = get_status()
422 if not _is_committed(status=status):
423 _LOG.error('\n'.join([
424 'Uncommitted changes to {prefix}* tags in notmuch',
426 "For a summary of changes, run 'nmbug status'",
427 "To save your changes, run 'nmbug commit' before merging/pull",
428 "To discard your changes, run 'nmbug checkout'",
429 ]).format(prefix=TAG_PREFIX))
433 def pull(repository=None, refspecs=None):
435 Pull (merge) remote repository changes to notmuch.
437 'pull' is equivalent to 'fetch' followed by 'merge'. We use the
438 Git-configured repository for your current branch
439 (branch.<name>.repository, likely 'origin', and
440 branch.<name>.merge, likely 'master').
443 if refspecs and not repository:
444 repository = _get_remote()
447 args.append(repository)
449 args.extend(refspecs)
450 with _tempfile.TemporaryDirectory(prefix='nmbug-pull.') as workdir:
456 additional_env={'GIT_WORK_TREE': workdir},
461 def merge(reference='@{upstream}'):
463 Merge changes from 'reference' into HEAD and load the result into notmuch.
465 The default reference is '@{upstream}'.
468 with _tempfile.TemporaryDirectory(prefix='nmbug-merge.') as workdir:
471 ['merge', reference]]:
474 additional_env={'GIT_WORK_TREE': workdir},
481 A simple wrapper for 'git log'.
483 After running 'nmbug fetch', you can inspect the changes with
484 'nmbug log HEAD..@{upstream}'.
486 # we don't want output trapping here, because we want the pager.
487 args = ['log', '--name-status', '--no-renames'] + list(args)
488 with _git(args=args, expect=(0, 1, -13)) as p:
492 def push(repository=None, refspecs=None):
493 "Push the local nmbug Git state to a remote repository."
494 if refspecs and not repository:
495 repository = _get_remote()
498 args.append(repository)
500 args.extend(refspecs)
501 _git(args=args, wait=True)
506 Show pending updates in notmuch or git repo.
508 Prints lines of the form
512 where n is a single character representing notmuch database status
516 Tag is present in notmuch database, but not committed to nmbug
517 (equivalently, tag has been deleted in nmbug repo, e.g. by a
518 pull, but not restored to notmuch database).
522 Tag is present in nmbug repo, but not restored to notmuch
523 database (equivalently, tag has been deleted in notmuch).
527 Message is unknown (missing from local notmuch database).
529 The second character (if present) represents a difference between
530 local and upstream branches. Typically 'nmbug fetch' needs to be
535 Tag is present in upstream, but not in the local Git branch.
539 Tag is present in local Git branch, but not upstream.
541 status = get_status()
542 # 'output' is a nested defaultdict for message status:
543 # * The outer dict is keyed by message id.
544 # * The inner dict is keyed by tag name.
545 # * The inner dict values are status strings (' a', 'Dd', ...).
546 output = _collections.defaultdict(
547 lambda : _collections.defaultdict(lambda : ' '))
548 for id, tags in status['added'].items():
550 output[id][tag] = 'A'
551 for id, tags in status['deleted'].items():
553 output[id][tag] = 'D'
554 for id, tags in status['missing'].items():
556 output[id][tag] = 'U'
558 for id, tag in _diff_refs(filter='A'):
559 output[id][tag] += 'a'
560 for id, tag in _diff_refs(filter='D'):
561 output[id][tag] += 'd'
562 for id, tag_status in sorted(output.items()):
563 for tag, status in sorted(tag_status.items()):
564 print('{status}\t{id}\t{tag}'.format(
565 status=status, id=id, tag=tag))
568 def _is_unmerged(ref='@{upstream}'):
570 (status, fetch_head, stderr) = _git(
571 args=['rev-parse', ref],
572 stdout=_subprocess.PIPE, stderr=_subprocess.PIPE, wait=True)
573 except SubprocessError as e:
574 if 'No upstream configured' in e.stderr:
577 (status, base, stderr) = _git(
578 args=['merge-base', 'HEAD', ref],
579 stdout=_subprocess.PIPE, wait=True)
580 return base != fetch_head
588 index = _index_tags()
589 maybe_deleted = _diff_index(index=index, filter='D')
590 for id, tags in maybe_deleted.items():
591 (_, stdout, stderr) = _spawn(
592 args=['notmuch', 'search', '--output=files', 'id:{0}'.format(id)],
593 stdout=_subprocess.PIPE,
596 status['deleted'][id] = tags
598 status['missing'][id] = tags
599 status['added'] = _diff_index(index=index, filter='A')
605 "Write notmuch tags to the nmbug.index."
606 path = _os.path.join(NMBGIT, 'nmbug.index')
607 query = ' '.join('tag:"{tag}"'.format(tag=tag) for tag in get_tags())
608 prefix = '+{0}'.format(_ENCODED_TAG_PREFIX)
610 args=['read-tree', '--empty'],
611 additional_env={'GIT_INDEX_FILE': path}, wait=True)
613 args=['notmuch', 'dump', '--format=batch-tag', '--', query],
614 stdout=_subprocess.PIPE) as notmuch:
616 args=['update-index', '--index-info'],
617 stdin=_subprocess.PIPE,
618 additional_env={'GIT_INDEX_FILE': path}) as git:
619 for line in notmuch.stdout:
620 if line.strip().startswith('#'):
622 (tags_string, id) = [_.strip() for _ in line.split(' -- id:')]
624 _unquote(tag[len(prefix):])
625 for tag in tags_string.split()
626 if tag.startswith(prefix)]
627 id = _xapian_unquote(string=id)
628 for line in _index_tags_for_message(
629 id=id, status='A', tags=tags):
630 git.stdin.write(line)
634 def _index_tags_for_message(id, status, tags):
636 Update the Git index to either create or delete an empty file.
638 Neither 'id' nor the tags in 'tags' should be encoded/escaped.
645 hash = '0000000000000000000000000000000000000000'
648 path = 'tags/{id}/{tag}'.format(
649 id=_hex_quote(string=id), tag=_hex_quote(string=tag))
650 yield '{mode} {hash}\t{path}\n'.format(mode=mode, hash=hash, path=path)
653 def _diff_index(index, filter):
655 Get an {id: {tag, ...}} dict for a given filter.
657 For example, use 'A' to find added tags, and 'D' to find deleted tags.
659 s = _collections.defaultdict(set)
662 'diff-index', '--cached', '--diff-filter', filter,
663 '--name-only', 'HEAD'],
664 additional_env={'GIT_INDEX_FILE': index},
665 stdout=_subprocess.PIPE) as p:
666 # Once we drop Python < 3.3, we can use 'yield from' here
667 for id, tag in _unpack_diff_lines(stream=p.stdout):
672 def _diff_refs(filter, a='HEAD', b='@{upstream}'):
674 args=['diff', '--diff-filter', filter, '--name-only', a, b],
675 stdout=_subprocess.PIPE) as p:
676 # Once we drop Python < 3.3, we can use 'yield from' here
677 for id, tag in _unpack_diff_lines(stream=p.stdout):
681 def _unpack_diff_lines(stream):
682 "Iterate through (id, tag) tuples in a diff stream."
684 match = _TAG_FILE_REGEX.match(line.strip())
687 'Invalid line in diff: {!r}'.format(line.strip()))
688 id = _unquote(match.group('id'))
689 tag = _unquote(match.group('tag'))
693 def _help(parser, command=None):
695 Show help for an nmbug command.
697 Because some folks prefer:
703 $ nmbug COMMAND --help
706 parser.parse_args([command, '--help'])
708 parser.parse_args(['--help'])
711 if __name__ == '__main__':
714 parser = argparse.ArgumentParser(
715 description=__doc__.strip(),
716 formatter_class=argparse.RawDescriptionHelpFormatter)
718 '-v', '--version', action='version',
719 version='%(prog)s {}'.format(__version__))
722 choices=['critical', 'error', 'warning', 'info', 'debug'],
723 help='Log verbosity. Defaults to {!r}.'.format(
724 _logging.getLevelName(_LOG.level).lower()))
726 help = _functools.partial(_help, parser=parser)
727 help.__doc__ = _help.__doc__
728 subparsers = parser.add_subparsers(
731 'For help on a particular command, run: '
732 "'%(prog)s ... <command> --help'."))
746 func = locals()[command]
747 doc = _textwrap.dedent(func.__doc__).strip().replace('%', '%%')
748 subparser = subparsers.add_parser(
750 help=doc.splitlines()[0],
752 formatter_class=argparse.RawDescriptionHelpFormatter)
753 subparser.set_defaults(func=func)
754 if command == 'archive':
755 subparser.add_argument(
756 'treeish', metavar='TREE-ISH', nargs='?', default='HEAD',
758 'The tree or commit to produce an archive for. Defaults '
760 subparser.add_argument(
761 'args', metavar='ARG', nargs='*',
763 "Argument passed through to 'git archive'. Set anything "
764 'before <tree-ish>, see git-archive(1) for details.'))
765 elif command == 'clone':
766 subparser.add_argument(
769 'The (possibly remote) repository to clone from. See the '
770 'URLS section of git-clone(1) for more information on '
771 'specifying repositories.'))
772 elif command == 'commit':
773 subparser.add_argument(
774 'message', metavar='MESSAGE', default='', nargs='?',
775 help='Text for the commit message.')
776 elif command == 'fetch':
777 subparser.add_argument(
778 'remote', metavar='REMOTE', nargs='?',
780 'Override the default configured in branch.<name>.remote '
781 'to fetch from a particular remote repository (e.g. '
783 elif command == 'help':
784 subparser.add_argument(
785 'command', metavar='COMMAND', nargs='?',
786 help='The command to show help for.')
787 elif command == 'log':
788 subparser.add_argument(
789 'args', metavar='ARG', nargs='*',
790 help="Additional argument passed through to 'git log'.")
791 elif command == 'merge':
792 subparser.add_argument(
793 'reference', metavar='REFERENCE', default='@{upstream}',
796 'Reference, usually other branch heads, to merge into '
797 "our branch. Defaults to '@{upstream}'."))
798 elif command == 'pull':
799 subparser.add_argument(
800 'repository', metavar='REPOSITORY', default=None, nargs='?',
802 'The "remote" repository that is the source of the pull. '
803 'This parameter can be either a URL (see the section GIT '
804 'URLS in git-pull(1)) or the name of a remote (see the '
805 'section REMOTES in git-pull(1)).'))
806 subparser.add_argument(
807 'refspecs', metavar='REFSPEC', default=None, nargs='*',
809 'Refspec (usually a branch name) to fetch and merge. See '
810 'the <refspec> entry in the OPTIONS section of '
811 'git-pull(1) for other possibilities.'))
812 elif command == 'push':
813 subparser.add_argument(
814 'repository', metavar='REPOSITORY', default=None, nargs='?',
816 'The "remote" repository that is the destination of the '
817 'push. This parameter can be either a URL (see the '
818 'section GIT URLS in git-push(1)) or the name of a remote '
819 '(see the section REMOTES in git-push(1)).'))
820 subparser.add_argument(
821 'refspecs', metavar='REFSPEC', default=None, nargs='*',
823 'Refspec (usually a branch name) to push. See '
824 'the <refspec> entry in the OPTIONS section of '
825 'git-push(1) for other possibilities.'))
827 args = parser.parse_args()
830 level = getattr(_logging, args.log_level.upper())
833 if not getattr(args, 'func', None):
837 if args.func == help:
838 arg_names = ['command']
840 (arg_names, varargs, varkw) = _inspect.getargs(args.func.__code__)
841 kwargs = {key: getattr(args, key) for key in arg_names if key in args}
844 except SubprocessError as e:
845 if _LOG.level == _logging.DEBUG:
846 raise # don't mask the traceback