3 # Copyright (c) 2011-2014 David Bremner <david@tethera.net>
4 # W. Trevor King <wking@tremily.us>
6 # This program is free software: you can redistribute it and/or modify
7 # it under the terms of the GNU General Public License as published by
8 # the Free Software Foundation, either version 3 of the License, or
9 # (at your option) any later version.
11 # This program is distributed in the hope that it will be useful,
12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 # GNU General Public License for more details.
16 # You should have received a copy of the GNU General Public License
17 # along with this program. If not, see https://www.gnu.org/licenses/ .
20 Manage notmuch tags with Git
22 Environment variables:
24 * NMBGIT specifies the location of the git repository used by nmbug.
25 If not specified $HOME/.nmbug is used.
26 * NMBPREFIX specifies the prefix in the notmuch database for tags of
27 interest to nmbug. If not specified 'notmuch::' is used.
30 from __future__ import print_function
31 from __future__ import unicode_literals
33 import codecs as _codecs
34 import collections as _collections
35 import functools as _functools
36 import inspect as _inspect
37 import locale as _locale
38 import logging as _logging
41 import shutil as _shutil
42 import subprocess as _subprocess
44 import tempfile as _tempfile
45 import textwrap as _textwrap
47 from urllib.parse import quote as _quote
48 from urllib.parse import unquote as _unquote
49 except ImportError: # Python 2
50 from urllib import quote as _quote
51 from urllib import unquote as _unquote
56 _LOG = _logging.getLogger('nmbug')
57 _LOG.setLevel(_logging.ERROR)
58 _LOG.addHandler(_logging.StreamHandler())
60 NMBGIT = _os.path.expanduser(
61 _os.getenv('NMBGIT', _os.path.join('~', '.nmbug')))
62 _NMBGIT = _os.path.join(NMBGIT, '.git')
63 if _os.path.isdir(_NMBGIT):
66 TAG_PREFIX = _os.getenv('NMBPREFIX', 'notmuch::')
67 _HEX_ESCAPE_REGEX = _re.compile('%[0-9A-F]{2}')
68 _TAG_FILE_REGEX = _re.compile('tags/(?P<id>[^/]*)/(?P<tag>[^/]*)')
70 # magic hash for Git (git hash-object -t blob /dev/null)
71 _EMPTYBLOB = 'e69de29bb2d1d6434b8b29ae775ad8c2e48c5391'
75 getattr(_tempfile, 'TemporaryDirectory')
76 except AttributeError: # Python < 3.2
77 class _TemporaryDirectory(object):
79 Fallback context manager for Python < 3.2
81 See PEP 343 for details on context managers [1].
83 [1]: https://www.python.org/dev/peps/pep-0343/
85 def __init__(self, **kwargs):
86 self.name = _tempfile.mkdtemp(**kwargs)
91 def __exit__(self, type, value, traceback):
92 _shutil.rmtree(self.name)
95 _tempfile.TemporaryDirectory = _TemporaryDirectory
98 def _hex_quote(string, safe='+@=:,'):
100 quote('abc def') -> 'abc%20def'.
102 Wrap urllib.parse.quote with additional safe characters (in
103 addition to letters, digits, and '_.-') and lowercase hex digits
104 (e.g. '%3a' instead of '%3A').
106 uppercase_escapes = _quote(string, safe)
107 return _HEX_ESCAPE_REGEX.sub(
108 lambda match: match.group(0).lower(),
112 _ENCODED_TAG_PREFIX = _hex_quote(TAG_PREFIX, safe='+@=,') # quote ':'
115 def _xapian_quote(string):
117 Quote a string for Xapian's QueryParser.
119 Xapian uses double-quotes for quoting strings. You can escape
120 internal quotes by repeating them [1,2,3].
122 [1]: https://trac.xapian.org/ticket/128#comment:2
123 [2]: https://trac.xapian.org/ticket/128#comment:17
124 [3]: https://trac.xapian.org/changeset/13823/svn
126 return '"{0}"'.format(string.replace('"', '""'))
129 def _xapian_unquote(string):
131 Unquote a Xapian-quoted string.
133 if string.startswith('"') and string.endswith('"'):
134 return string[1:-1].replace('""', '"')
138 class SubprocessError(RuntimeError):
139 "A subprocess exited with a nonzero status"
140 def __init__(self, args, status, stdout=None, stderr=None):
144 msg = '{args} exited with {status}'.format(args=args, status=status)
146 msg = '{msg}: {stderr}'.format(msg=msg, stderr=stderr)
147 super(SubprocessError, self).__init__(msg)
150 class _SubprocessContextManager(object):
152 PEP 343 context manager for subprocesses.
154 'expect' holds a tuple of acceptable exit codes, otherwise we'll
155 raise a SubprocessError in __exit__.
157 def __init__(self, process, args, expect=(0,)):
158 self._process = process
160 self._expect = expect
165 def __exit__(self, type, value, traceback):
166 for name in ['stdin', 'stdout', 'stderr']:
167 stream = getattr(self._process, name)
170 setattr(self._process, name, None)
171 status = self._process.wait()
173 'collect {args} with status {status} (expected {expect})'.format(
174 args=self._args, status=status, expect=self._expect))
175 if status not in self._expect:
176 raise SubprocessError(args=self._args, status=status)
179 return self._process.wait()
182 def _spawn(args, input=None, additional_env=None, wait=False, stdin=None,
183 stdout=None, stderr=None, encoding=_locale.getpreferredencoding(),
184 expect=(0,), **kwargs):
185 """Spawn a subprocess, and optionally wait for it to finish.
187 This wrapper around subprocess.Popen has two modes, depending on
188 the truthiness of 'wait'. If 'wait' is true, we use p.communicate
189 internally to write 'input' to the subprocess's stdin and read
190 from it's stdout/stderr. If 'wait' is False, we return a
191 _SubprocessContextManager instance for fancier handling
192 (e.g. piping between processes).
194 For 'wait' calls when you want to write to the subprocess's stdin,
195 you only need to set 'input' to your content. When 'input' is not
196 None but 'stdin' is, we'll automatically set 'stdin' to PIPE
197 before calling Popen. This avoids having the subprocess
198 accidentally inherit the launching process's stdin.
200 _LOG.debug('spawn {args} (additional env. var.: {env})'.format(
201 args=args, env=additional_env))
202 if not stdin and input is not None:
203 stdin = _subprocess.PIPE
205 if not kwargs.get('env'):
206 kwargs['env'] = dict(_os.environ)
207 kwargs['env'].update(additional_env)
208 p = _subprocess.Popen(
209 args, stdin=stdin, stdout=stdout, stderr=stderr, **kwargs)
211 if hasattr(input, 'encode'):
212 input = input.encode(encoding)
213 (stdout, stderr) = p.communicate(input=input)
216 'collect {args} with status {status} (expected {expect})'.format(
217 args=args, status=status, expect=expect))
218 if stdout is not None:
219 stdout = stdout.decode(encoding)
220 if stderr is not None:
221 stderr = stderr.decode(encoding)
222 if status not in expect:
223 raise SubprocessError(
224 args=args, status=status, stdout=stdout, stderr=stderr)
225 return (status, stdout, stderr)
226 if p.stdin and not stdin:
230 p.stdin = _codecs.getwriter(encoding=encoding)(stream=p.stdin)
231 stream_reader = _codecs.getreader(encoding=encoding)
233 p.stdout = stream_reader(stream=p.stdout)
235 p.stderr = stream_reader(stream=p.stderr)
236 return _SubprocessContextManager(args=args, process=p, expect=expect)
239 def _git(args, **kwargs):
240 args = ['git', '--git-dir', NMBGIT] + list(args)
241 return _spawn(args=args, **kwargs)
244 def _get_current_branch():
245 """Get the name of the current branch.
247 Return 'None' if we're not on a branch.
250 (status, branch, stderr) = _git(
251 args=['symbolic-ref', '--short', 'HEAD'],
252 stdout=_subprocess.PIPE, stderr=_subprocess.PIPE, wait=True)
253 except SubprocessError as e:
254 if 'not a symbolic ref' in e:
257 return branch.strip()
261 "Get the default remote for the current branch."
262 local_branch = _get_current_branch()
263 (status, remote, stderr) = _git(
264 args=['config', 'branch.{0}.remote'.format(local_branch)],
265 stdout=_subprocess.PIPE, wait=True)
266 return remote.strip()
269 def get_tags(prefix=None):
270 "Get a list of tags with a given prefix."
273 (status, stdout, stderr) = _spawn(
274 args=['notmuch', 'search', '--output=tags', '*'],
275 stdout=_subprocess.PIPE, wait=True)
276 return [tag for tag in stdout.splitlines() if tag.startswith(prefix)]
279 def archive(treeish='HEAD', args=()):
281 Dump a tar archive of the current nmbug tag set.
285 Each tag $tag for message with Message-Id $id is written to
288 tags/encode($id)/encode($tag)
290 The encoding preserves alphanumerics, and the characters
291 "+-_@=.:," (not the quotes). All other octets are replaced with
292 '%' followed by a two digit hex number.
294 _git(args=['archive', treeish] + list(args), wait=True)
297 def clone(repository):
299 Create a local nmbug repository from a remote source.
301 This wraps 'git clone', adding some options to avoid creating a
302 working tree while preserving remote-tracking branches and
305 with _tempfile.TemporaryDirectory(prefix='nmbug-clone.') as workdir:
308 'git', 'clone', '--no-checkout', '--separate-git-dir', NMBGIT,
309 repository, workdir],
311 _git(args=['config', '--unset', 'core.worktree'], wait=True)
312 _git(args=['config', 'core.bare', 'true'], wait=True)
313 _git(args=['branch', 'config', 'origin/config'], wait=True)
316 def _is_committed(status):
317 return len(status['added']) + len(status['deleted']) == 0
320 def commit(treeish='HEAD', message=None):
322 Commit prefix-matching tags from the notmuch database to Git.
324 status = get_status()
326 if _is_committed(status=status):
327 _LOG.warning('Nothing to commit')
330 _git(args=['read-tree', '--empty'], wait=True)
331 _git(args=['read-tree', treeish], wait=True)
333 _update_index(status=status)
336 stdout=_subprocess.PIPE,
338 (_, parent, _) = _git(
339 args=['rev-parse', treeish],
340 stdout=_subprocess.PIPE,
342 (_, commit, _) = _git(
343 args=['commit-tree', tree.strip(), '-p', parent.strip()],
345 stdout=_subprocess.PIPE,
348 args=['update-ref', treeish, commit.strip()],
349 stdout=_subprocess.PIPE,
351 except Exception as e:
352 _git(args=['read-tree', '--empty'], wait=True)
353 _git(args=['read-tree', treeish], wait=True)
356 def _update_index(status):
358 args=['update-index', '--index-info'],
359 stdin=_subprocess.PIPE) as p:
360 for id, tags in status['deleted'].items():
361 for line in _index_tags_for_message(id=id, status='D', tags=tags):
363 for id, tags in status['added'].items():
364 for line in _index_tags_for_message(id=id, status='A', tags=tags):
368 def fetch(remote=None):
370 Fetch changes from the remote repository.
372 See 'merge' to bring those changes into notmuch.
377 _git(args=args, wait=True)
382 Update the notmuch database from Git.
384 This is mainly useful to discard your changes in notmuch relative
387 status = get_status()
389 args=['notmuch', 'tag', '--batch'], stdin=_subprocess.PIPE) as p:
390 for id, tags in status['added'].items():
391 p.stdin.write(_batch_line(action='-', id=id, tags=tags))
392 for id, tags in status['deleted'].items():
393 p.stdin.write(_batch_line(action='+', id=id, tags=tags))
396 def _batch_line(action, id, tags):
398 'notmuch tag --batch' line for adding/removing tags.
400 Set 'action' to '-' to remove a tag or '+' to add the tags to a
403 tag_string = ' '.join(
404 '{action}{prefix}{tag}'.format(
405 action=action, prefix=_ENCODED_TAG_PREFIX, tag=_hex_quote(tag))
407 line = '{tags} -- id:{id}\n'.format(
408 tags=tag_string, id=_xapian_quote(string=id))
412 def _insist_committed():
413 "Die if the the notmuch tags don't match the current HEAD."
414 status = get_status()
415 if not _is_committed(status=status):
416 _LOG.error('\n'.join([
417 'Uncommitted changes to {prefix}* tags in notmuch',
419 "For a summary of changes, run 'nmbug status'",
420 "To save your changes, run 'nmbug commit' before merging/pull",
421 "To discard your changes, run 'nmbug checkout'",
422 ]).format(prefix=TAG_PREFIX))
426 def pull(repository=None, refspecs=None):
428 Pull (merge) remote repository changes to notmuch.
430 'pull' is equivalent to 'fetch' followed by 'merge'. We use the
431 Git-configured repository for your current branch
432 (branch.<name>.repository, likely 'origin', and
433 branch.<name>.merge, likely 'master').
436 if refspecs and not repository:
437 repository = _get_remote()
440 args.append(repository)
442 args.extend(refspecs)
443 with _tempfile.TemporaryDirectory(prefix='nmbug-pull.') as workdir:
449 additional_env={'GIT_WORK_TREE': workdir},
454 def merge(reference='@{upstream}'):
456 Merge changes from 'reference' into HEAD and load the result into notmuch.
458 The default reference is '@{upstream}'.
461 with _tempfile.TemporaryDirectory(prefix='nmbug-merge.') as workdir:
464 ['merge', reference]]:
467 additional_env={'GIT_WORK_TREE': workdir},
474 A simple wrapper for 'git log'.
476 After running 'nmbug fetch', you can inspect the changes with
477 'nmbug log HEAD..@{upstream}'.
479 # we don't want output trapping here, because we want the pager.
480 args = ['log', '--name-status', '--no-renames'] + list(args)
481 with _git(args=args, expect=(0, 1, -13)) as p:
485 def push(repository=None, refspecs=None):
486 "Push the local nmbug Git state to a remote repository."
487 if refspecs and not repository:
488 repository = _get_remote()
491 args.append(repository)
493 args.extend(refspecs)
494 _git(args=args, wait=True)
499 Show pending updates in notmuch or git repo.
501 Prints lines of the form
505 where n is a single character representing notmuch database status
509 Tag is present in notmuch database, but not committed to nmbug
510 (equivalently, tag has been deleted in nmbug repo, e.g. by a
511 pull, but not restored to notmuch database).
515 Tag is present in nmbug repo, but not restored to notmuch
516 database (equivalently, tag has been deleted in notmuch).
520 Message is unknown (missing from local notmuch database).
522 The second character (if present) represents a difference between
523 local and upstream branches. Typically 'nmbug fetch' needs to be
528 Tag is present in upstream, but not in the local Git branch.
532 Tag is present in local Git branch, but not upstream.
534 status = get_status()
535 # 'output' is a nested defaultdict for message status:
536 # * The outer dict is keyed by message id.
537 # * The inner dict is keyed by tag name.
538 # * The inner dict values are status strings (' a', 'Dd', ...).
539 output = _collections.defaultdict(
540 lambda : _collections.defaultdict(lambda : ' '))
541 for id, tags in status['added'].items():
543 output[id][tag] = 'A'
544 for id, tags in status['deleted'].items():
546 output[id][tag] = 'D'
547 for id, tags in status['missing'].items():
549 output[id][tag] = 'U'
551 for id, tag in _diff_refs(filter='A'):
552 output[id][tag] += 'a'
553 for id, tag in _diff_refs(filter='D'):
554 output[id][tag] += 'd'
555 for id, tag_status in sorted(output.items()):
556 for tag, status in sorted(tag_status.items()):
557 print('{status}\t{id}\t{tag}'.format(
558 status=status, id=id, tag=tag))
561 def _is_unmerged(ref='@{upstream}'):
563 (status, fetch_head, stderr) = _git(
564 args=['rev-parse', ref],
565 stdout=_subprocess.PIPE, stderr=_subprocess.PIPE, wait=True)
566 except SubprocessError as e:
567 if 'No upstream configured' in e.stderr:
570 (status, base, stderr) = _git(
571 args=['merge-base', 'HEAD', ref],
572 stdout=_subprocess.PIPE, wait=True)
573 return base != fetch_head
581 index = _index_tags()
582 maybe_deleted = _diff_index(index=index, filter='D')
583 for id, tags in maybe_deleted.items():
584 (_, stdout, stderr) = _spawn(
585 args=['notmuch', 'search', '--output=files', 'id:{0}'.format(id)],
586 stdout=_subprocess.PIPE,
589 status['deleted'][id] = tags
591 status['missing'][id] = tags
592 status['added'] = _diff_index(index=index, filter='A')
598 "Write notmuch tags to the nmbug.index."
599 path = _os.path.join(NMBGIT, 'nmbug.index')
600 query = ' '.join('tag:"{tag}"'.format(tag=tag) for tag in get_tags())
601 prefix = '+{0}'.format(_ENCODED_TAG_PREFIX)
603 args=['read-tree', '--empty'],
604 additional_env={'GIT_INDEX_FILE': path}, wait=True)
606 args=['notmuch', 'dump', '--format=batch-tag', '--', query],
607 stdout=_subprocess.PIPE) as notmuch:
609 args=['update-index', '--index-info'],
610 stdin=_subprocess.PIPE,
611 additional_env={'GIT_INDEX_FILE': path}) as git:
612 for line in notmuch.stdout:
613 if line.strip().startswith('#'):
615 (tags_string, id) = [_.strip() for _ in line.split(' -- id:')]
617 _unquote(tag[len(prefix):])
618 for tag in tags_string.split()
619 if tag.startswith(prefix)]
620 id = _xapian_unquote(string=id)
621 for line in _index_tags_for_message(
622 id=id, status='A', tags=tags):
623 git.stdin.write(line)
627 def _index_tags_for_message(id, status, tags):
629 Update the Git index to either create or delete an empty file.
631 Neither 'id' nor the tags in 'tags' should be encoded/escaped.
638 hash = '0000000000000000000000000000000000000000'
641 path = 'tags/{id}/{tag}'.format(
642 id=_hex_quote(string=id), tag=_hex_quote(string=tag))
643 yield '{mode} {hash}\t{path}\n'.format(mode=mode, hash=hash, path=path)
646 def _diff_index(index, filter):
648 Get an {id: {tag, ...}} dict for a given filter.
650 For example, use 'A' to find added tags, and 'D' to find deleted tags.
652 s = _collections.defaultdict(set)
655 'diff-index', '--cached', '--diff-filter', filter,
656 '--name-only', 'HEAD'],
657 additional_env={'GIT_INDEX_FILE': index},
658 stdout=_subprocess.PIPE) as p:
659 # Once we drop Python < 3.3, we can use 'yield from' here
660 for id, tag in _unpack_diff_lines(stream=p.stdout):
665 def _diff_refs(filter, a='HEAD', b='@{upstream}'):
667 args=['diff', '--diff-filter', filter, '--name-only', a, b],
668 stdout=_subprocess.PIPE) as p:
669 # Once we drop Python < 3.3, we can use 'yield from' here
670 for id, tag in _unpack_diff_lines(stream=p.stdout):
674 def _unpack_diff_lines(stream):
675 "Iterate through (id, tag) tuples in a diff stream."
677 match = _TAG_FILE_REGEX.match(line.strip())
680 'Invalid line in diff: {!r}'.format(line.strip()))
681 id = _unquote(match.group('id'))
682 tag = _unquote(match.group('tag'))
686 def _help(parser, command=None):
688 Show help for an nmbug command.
690 Because some folks prefer:
696 $ nmbug COMMAND --help
699 parser.parse_args([command, '--help'])
701 parser.parse_args(['--help'])
704 if __name__ == '__main__':
707 parser = argparse.ArgumentParser(
708 description=__doc__.strip(),
709 formatter_class=argparse.RawDescriptionHelpFormatter)
711 '-v', '--version', action='version',
712 version='%(prog)s {}'.format(__version__))
715 choices=['critical', 'error', 'warning', 'info', 'debug'],
716 help='Log verbosity. Defaults to {!r}.'.format(
717 _logging.getLevelName(_LOG.level).lower()))
719 help = _functools.partial(_help, parser=parser)
720 help.__doc__ = _help.__doc__
721 subparsers = parser.add_subparsers(
724 'For help on a particular command, run: '
725 "'%(prog)s ... <command> --help'."))
739 func = locals()[command]
740 doc = _textwrap.dedent(func.__doc__).strip().replace('%', '%%')
741 subparser = subparsers.add_parser(
743 help=doc.splitlines()[0],
745 formatter_class=argparse.RawDescriptionHelpFormatter)
746 subparser.set_defaults(func=func)
747 if command == 'archive':
748 subparser.add_argument(
749 'treeish', metavar='TREE-ISH', nargs='?', default='HEAD',
751 'The tree or commit to produce an archive for. Defaults '
753 subparser.add_argument(
754 'args', metavar='ARG', nargs='*',
756 "Argument passed through to 'git archive'. Set anything "
757 'before <tree-ish>, see git-archive(1) for details.'))
758 elif command == 'clone':
759 subparser.add_argument(
762 'The (possibly remote) repository to clone from. See the '
763 'URLS section of git-clone(1) for more information on '
764 'specifying repositories.'))
765 elif command == 'commit':
766 subparser.add_argument(
767 'message', metavar='MESSAGE', default='', nargs='?',
768 help='Text for the commit message.')
769 elif command == 'fetch':
770 subparser.add_argument(
771 'remote', metavar='REMOTE', nargs='?',
773 'Override the default configured in branch.<name>.remote '
774 'to fetch from a particular remote repository (e.g. '
776 elif command == 'help':
777 subparser.add_argument(
778 'command', metavar='COMMAND', nargs='?',
779 help='The command to show help for.')
780 elif command == 'log':
781 subparser.add_argument(
782 'args', metavar='ARG', nargs='*',
783 help="Additional argument passed through to 'git log'.")
784 elif command == 'merge':
785 subparser.add_argument(
786 'reference', metavar='REFERENCE', default='@{upstream}',
789 'Reference, usually other branch heads, to merge into '
790 "our branch. Defaults to '@{upstream}'."))
791 elif command == 'pull':
792 subparser.add_argument(
793 'repository', metavar='REPOSITORY', default=None, nargs='?',
795 'The "remote" repository that is the source of the pull. '
796 'This parameter can be either a URL (see the section GIT '
797 'URLS in git-pull(1)) or the name of a remote (see the '
798 'section REMOTES in git-pull(1)).'))
799 subparser.add_argument(
800 'refspecs', metavar='REFSPEC', default=None, nargs='*',
802 'Refspec (usually a branch name) to fetch and merge. See '
803 'the <refspec> entry in the OPTIONS section of '
804 'git-pull(1) for other possibilities.'))
805 elif command == 'push':
806 subparser.add_argument(
807 'repository', metavar='REPOSITORY', default=None, nargs='?',
809 'The "remote" repository that is the destination of the '
810 'push. This parameter can be either a URL (see the '
811 'section GIT URLS in git-push(1)) or the name of a remote '
812 '(see the section REMOTES in git-push(1)).'))
813 subparser.add_argument(
814 'refspecs', metavar='REFSPEC', default=None, nargs='*',
816 'Refspec (usually a branch name) to push. See '
817 'the <refspec> entry in the OPTIONS section of '
818 'git-push(1) for other possibilities.'))
820 args = parser.parse_args()
823 level = getattr(_logging, args.log_level.upper())
826 if not getattr(args, 'func', None):
830 if args.func == help:
831 arg_names = ['command']
833 (arg_names, varargs, varkw) = _inspect.getargs(args.func.__code__)
834 kwargs = {key: getattr(args, key) for key in arg_names if key in args}
837 except SubprocessError as e:
838 if _LOG.level == _logging.DEBUG:
839 raise # don't mask the traceback