3 # Copyright (c) 2011-2014 David Bremner <david@tethera.net>
4 # W. Trevor King <wking@tremily.us>
6 # This program is free software: you can redistribute it and/or modify
7 # it under the terms of the GNU General Public License as published by
8 # the Free Software Foundation, either version 3 of the License, or
9 # (at your option) any later version.
11 # This program is distributed in the hope that it will be useful,
12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 # GNU General Public License for more details.
16 # You should have received a copy of the GNU General Public License
17 # along with this program. If not, see https://www.gnu.org/licenses/ .
20 Manage notmuch tags with Git
22 Environment variables:
24 * NMBGIT specifies the location of the git repository used by nmbug.
25 If not specified $HOME/.nmbug is used.
26 * NMBPREFIX specifies the prefix in the notmuch database for tags of
27 interest to nmbug. If not specified 'notmuch::' is used.
30 from __future__ import print_function
31 from __future__ import unicode_literals
33 import codecs as _codecs
34 import collections as _collections
35 import functools as _functools
36 import inspect as _inspect
37 import locale as _locale
38 import logging as _logging
41 import shutil as _shutil
42 import subprocess as _subprocess
44 import tempfile as _tempfile
45 import textwrap as _textwrap
46 from urllib.parse import quote as _quote
47 from urllib.parse import unquote as _unquote
49 _LOG = _logging.getLogger('nmbug')
50 _LOG.setLevel(_logging.WARNING)
51 _LOG.addHandler(_logging.StreamHandler())
56 _HEX_ESCAPE_REGEX = _re.compile('%[0-9A-F]{2}')
57 _TAG_DIRECTORY = 'tags/'
58 _TAG_FILE_REGEX = _re.compile(_TAG_DIRECTORY + '(?P<id>[^/]*)/(?P<tag>[^/]*)')
60 # magic hash for Git (git hash-object -t blob /dev/null)
61 _EMPTYBLOB = 'e69de29bb2d1d6434b8b29ae775ad8c2e48c5391'
63 def _hex_quote(string, safe='+@=:,'):
65 quote('abc def') -> 'abc%20def'.
67 Wrap urllib.parse.quote with additional safe characters (in
68 addition to letters, digits, and '_.-') and lowercase hex digits
69 (e.g. '%3a' instead of '%3A').
71 uppercase_escapes = _quote(string, safe)
72 return _HEX_ESCAPE_REGEX.sub(
73 lambda match: match.group(0).lower(),
76 def _xapian_quote(string):
78 Quote a string for Xapian's QueryParser.
80 Xapian uses double-quotes for quoting strings. You can escape
81 internal quotes by repeating them [1,2,3].
83 [1]: https://trac.xapian.org/ticket/128#comment:2
84 [2]: https://trac.xapian.org/ticket/128#comment:17
85 [3]: https://trac.xapian.org/changeset/13823/svn
87 return '"{0}"'.format(string.replace('"', '""'))
90 def _xapian_unquote(string):
92 Unquote a Xapian-quoted string.
94 if string.startswith('"') and string.endswith('"'):
95 return string[1:-1].replace('""', '"')
99 class SubprocessError(RuntimeError):
100 "A subprocess exited with a nonzero status"
101 def __init__(self, args, status, stdout=None, stderr=None):
105 msg = '{args} exited with {status}'.format(args=args, status=status)
107 msg = '{msg}: {stderr}'.format(msg=msg, stderr=stderr)
108 super(SubprocessError, self).__init__(msg)
111 class _SubprocessContextManager(object):
113 PEP 343 context manager for subprocesses.
115 'expect' holds a tuple of acceptable exit codes, otherwise we'll
116 raise a SubprocessError in __exit__.
118 def __init__(self, process, args, expect=(0,)):
119 self._process = process
121 self._expect = expect
126 def __exit__(self, type, value, traceback):
127 for name in ['stdin', 'stdout', 'stderr']:
128 stream = getattr(self._process, name)
131 setattr(self._process, name, None)
132 status = self._process.wait()
134 'collect {args} with status {status} (expected {expect})'.format(
135 args=self._args, status=status, expect=self._expect))
136 if status not in self._expect:
137 raise SubprocessError(args=self._args, status=status)
140 return self._process.wait()
143 def _spawn(args, input=None, additional_env=None, wait=False, stdin=None,
144 stdout=None, stderr=None, encoding=_locale.getpreferredencoding(),
145 expect=(0,), **kwargs):
146 """Spawn a subprocess, and optionally wait for it to finish.
148 This wrapper around subprocess.Popen has two modes, depending on
149 the truthiness of 'wait'. If 'wait' is true, we use p.communicate
150 internally to write 'input' to the subprocess's stdin and read
151 from it's stdout/stderr. If 'wait' is False, we return a
152 _SubprocessContextManager instance for fancier handling
153 (e.g. piping between processes).
155 For 'wait' calls when you want to write to the subprocess's stdin,
156 you only need to set 'input' to your content. When 'input' is not
157 None but 'stdin' is, we'll automatically set 'stdin' to PIPE
158 before calling Popen. This avoids having the subprocess
159 accidentally inherit the launching process's stdin.
161 _LOG.debug('spawn {args} (additional env. var.: {env})'.format(
162 args=args, env=additional_env))
163 if not stdin and input is not None:
164 stdin = _subprocess.PIPE
166 if not kwargs.get('env'):
167 kwargs['env'] = dict(_os.environ)
168 kwargs['env'].update(additional_env)
169 p = _subprocess.Popen(
170 args, stdin=stdin, stdout=stdout, stderr=stderr, **kwargs)
172 if hasattr(input, 'encode'):
173 input = input.encode(encoding)
174 (stdout, stderr) = p.communicate(input=input)
177 'collect {args} with status {status} (expected {expect})'.format(
178 args=args, status=status, expect=expect))
179 if stdout is not None:
180 stdout = stdout.decode(encoding)
181 if stderr is not None:
182 stderr = stderr.decode(encoding)
183 if status not in expect:
184 raise SubprocessError(
185 args=args, status=status, stdout=stdout, stderr=stderr)
186 return (status, stdout, stderr)
187 if p.stdin and not stdin:
191 p.stdin = _codecs.getwriter(encoding=encoding)(stream=p.stdin)
192 stream_reader = _codecs.getreader(encoding=encoding)
194 p.stdout = stream_reader(stream=p.stdout)
196 p.stderr = stream_reader(stream=p.stderr)
197 return _SubprocessContextManager(args=args, process=p, expect=expect)
200 def _git(args, **kwargs):
201 args = ['git', '--git-dir', NMBGIT] + list(args)
202 return _spawn(args=args, **kwargs)
205 def _get_current_branch():
206 """Get the name of the current branch.
208 Return 'None' if we're not on a branch.
211 (status, branch, stderr) = _git(
212 args=['symbolic-ref', '--short', 'HEAD'],
213 stdout=_subprocess.PIPE, stderr=_subprocess.PIPE, wait=True)
214 except SubprocessError as e:
215 if 'not a symbolic ref' in e:
218 return branch.strip()
222 "Get the default remote for the current branch."
223 local_branch = _get_current_branch()
224 (status, remote, stderr) = _git(
225 args=['config', 'branch.{0}.remote'.format(local_branch)],
226 stdout=_subprocess.PIPE, wait=True)
227 return remote.strip()
230 def get_tags(prefix=None):
231 "Get a list of tags with a given prefix."
234 (status, stdout, stderr) = _spawn(
235 args=['notmuch', 'search', '--output=tags', '*'],
236 stdout=_subprocess.PIPE, wait=True)
237 return [tag for tag in stdout.splitlines() if tag.startswith(prefix)]
240 def archive(treeish='HEAD', args=()):
242 Dump a tar archive of the current nmbug tag set.
246 Each tag $tag for message with Message-Id $id is written to
249 tags/encode($id)/encode($tag)
251 The encoding preserves alphanumerics, and the characters
252 "+-_@=.:," (not the quotes). All other octets are replaced with
253 '%' followed by a two digit hex number.
255 _git(args=['archive', treeish] + list(args), wait=True)
258 def clone(repository):
260 Create a local nmbug repository from a remote source.
262 This wraps 'git clone', adding some options to avoid creating a
263 working tree while preserving remote-tracking branches and
266 with _tempfile.TemporaryDirectory(prefix='nmbug-clone.') as workdir:
269 'git', 'clone', '--no-checkout', '--separate-git-dir', NMBGIT,
270 repository, workdir],
272 _git(args=['config', '--unset', 'core.worktree'], wait=True, expect=(0, 5))
273 _git(args=['config', 'core.bare', 'true'], wait=True)
274 (status, stdout, stderr) = _git(args=['show-ref', '--verify',
276 'refs/remotes/origin/config'],
280 _git(args=['branch', 'config', 'origin/config'], wait=True)
281 existing_tags = get_tags()
284 'Not checking out to avoid clobbering existing tags: {}'.format(
285 ', '.join(existing_tags)))
290 def _is_committed(status):
291 return len(status['added']) + len(status['deleted']) == 0
294 def commit(treeish='HEAD', message=None):
296 Commit prefix-matching tags from the notmuch database to Git.
298 status = get_status()
300 if _is_committed(status=status):
301 _LOG.warning('Nothing to commit')
304 _git(args=['read-tree', '--empty'], wait=True)
305 _git(args=['read-tree', treeish], wait=True)
307 _update_index(status=status)
310 stdout=_subprocess.PIPE,
312 (_, parent, _) = _git(
313 args=['rev-parse', treeish],
314 stdout=_subprocess.PIPE,
316 (_, commit, _) = _git(
317 args=['commit-tree', tree.strip(), '-p', parent.strip()],
319 stdout=_subprocess.PIPE,
322 args=['update-ref', treeish, commit.strip()],
323 stdout=_subprocess.PIPE,
325 except Exception as e:
326 _git(args=['read-tree', '--empty'], wait=True)
327 _git(args=['read-tree', treeish], wait=True)
330 def _update_index(status):
332 args=['update-index', '--index-info'],
333 stdin=_subprocess.PIPE) as p:
334 for id, tags in status['deleted'].items():
335 for line in _index_tags_for_message(id=id, status='D', tags=tags):
337 for id, tags in status['added'].items():
338 for line in _index_tags_for_message(id=id, status='A', tags=tags):
342 def fetch(remote=None):
344 Fetch changes from the remote repository.
346 See 'merge' to bring those changes into notmuch.
351 _git(args=args, wait=True)
356 Update the notmuch database from Git.
358 This is mainly useful to discard your changes in notmuch relative
361 status = get_status()
363 args=['notmuch', 'tag', '--batch'], stdin=_subprocess.PIPE) as p:
364 for id, tags in status['added'].items():
365 p.stdin.write(_batch_line(action='-', id=id, tags=tags))
366 for id, tags in status['deleted'].items():
367 p.stdin.write(_batch_line(action='+', id=id, tags=tags))
370 def _batch_line(action, id, tags):
372 'notmuch tag --batch' line for adding/removing tags.
374 Set 'action' to '-' to remove a tag or '+' to add the tags to a
377 tag_string = ' '.join(
378 '{action}{prefix}{tag}'.format(
379 action=action, prefix=_ENCODED_TAG_PREFIX, tag=_hex_quote(tag))
381 line = '{tags} -- id:{id}\n'.format(
382 tags=tag_string, id=_xapian_quote(string=id))
386 def _insist_committed():
387 "Die if the the notmuch tags don't match the current HEAD."
388 status = get_status()
389 if not _is_committed(status=status):
390 _LOG.error('\n'.join([
391 'Uncommitted changes to {prefix}* tags in notmuch',
393 "For a summary of changes, run 'nmbug status'",
394 "To save your changes, run 'nmbug commit' before merging/pull",
395 "To discard your changes, run 'nmbug checkout'",
396 ]).format(prefix=TAG_PREFIX))
400 def pull(repository=None, refspecs=None):
402 Pull (merge) remote repository changes to notmuch.
404 'pull' is equivalent to 'fetch' followed by 'merge'. We use the
405 Git-configured repository for your current branch
406 (branch.<name>.repository, likely 'origin', and
407 branch.<name>.merge, likely 'master').
410 if refspecs and not repository:
411 repository = _get_remote()
414 args.append(repository)
416 args.extend(refspecs)
417 with _tempfile.TemporaryDirectory(prefix='nmbug-pull.') as workdir:
423 additional_env={'GIT_WORK_TREE': workdir},
428 def merge(reference='@{upstream}'):
430 Merge changes from 'reference' into HEAD and load the result into notmuch.
432 The default reference is '@{upstream}'.
435 with _tempfile.TemporaryDirectory(prefix='nmbug-merge.') as workdir:
438 ['merge', reference]]:
441 additional_env={'GIT_WORK_TREE': workdir},
448 A simple wrapper for 'git log'.
450 After running 'nmbug fetch', you can inspect the changes with
451 'nmbug log HEAD..@{upstream}'.
453 # we don't want output trapping here, because we want the pager.
454 args = ['log', '--name-status', '--no-renames'] + list(args)
455 with _git(args=args, expect=(0, 1, -13)) as p:
459 def push(repository=None, refspecs=None):
460 "Push the local nmbug Git state to a remote repository."
461 if refspecs and not repository:
462 repository = _get_remote()
465 args.append(repository)
467 args.extend(refspecs)
468 _git(args=args, wait=True)
473 Show pending updates in notmuch or git repo.
475 Prints lines of the form
479 where n is a single character representing notmuch database status
483 Tag is present in notmuch database, but not committed to nmbug
484 (equivalently, tag has been deleted in nmbug repo, e.g. by a
485 pull, but not restored to notmuch database).
489 Tag is present in nmbug repo, but not restored to notmuch
490 database (equivalently, tag has been deleted in notmuch).
494 Message is unknown (missing from local notmuch database).
496 The second character (if present) represents a difference between
497 local and upstream branches. Typically 'nmbug fetch' needs to be
502 Tag is present in upstream, but not in the local Git branch.
506 Tag is present in local Git branch, but not upstream.
508 status = get_status()
509 # 'output' is a nested defaultdict for message status:
510 # * The outer dict is keyed by message id.
511 # * The inner dict is keyed by tag name.
512 # * The inner dict values are status strings (' a', 'Dd', ...).
513 output = _collections.defaultdict(
514 lambda : _collections.defaultdict(lambda : ' '))
515 for id, tags in status['added'].items():
517 output[id][tag] = 'A'
518 for id, tags in status['deleted'].items():
520 output[id][tag] = 'D'
521 for id, tags in status['missing'].items():
523 output[id][tag] = 'U'
525 for id, tag in _diff_refs(filter='A'):
526 output[id][tag] += 'a'
527 for id, tag in _diff_refs(filter='D'):
528 output[id][tag] += 'd'
529 for id, tag_status in sorted(output.items()):
530 for tag, status in sorted(tag_status.items()):
531 print('{status}\t{id}\t{tag}'.format(
532 status=status, id=id, tag=tag))
535 def _is_unmerged(ref='@{upstream}'):
537 (status, fetch_head, stderr) = _git(
538 args=['rev-parse', ref],
539 stdout=_subprocess.PIPE, stderr=_subprocess.PIPE, wait=True)
540 except SubprocessError as e:
541 if 'No upstream configured' in e.stderr:
544 (status, base, stderr) = _git(
545 args=['merge-base', 'HEAD', ref],
546 stdout=_subprocess.PIPE, wait=True)
547 return base != fetch_head
555 index = _index_tags()
556 maybe_deleted = _diff_index(index=index, filter='D')
557 for id, tags in maybe_deleted.items():
558 (_, stdout, stderr) = _spawn(
559 args=['notmuch', 'search', '--output=files', 'id:{0}'.format(id)],
560 stdout=_subprocess.PIPE,
563 status['deleted'][id] = tags
565 status['missing'][id] = tags
566 status['added'] = _diff_index(index=index, filter='A')
572 "Write notmuch tags to the nmbug.index."
573 path = _os.path.join(NMBGIT, 'nmbug.index')
574 query = ' '.join('tag:"{tag}"'.format(tag=tag) for tag in get_tags())
575 prefix = '+{0}'.format(_ENCODED_TAG_PREFIX)
577 args=['read-tree', '--empty'],
578 additional_env={'GIT_INDEX_FILE': path}, wait=True)
580 args=['notmuch', 'dump', '--format=batch-tag', '--', query],
581 stdout=_subprocess.PIPE) as notmuch:
583 args=['update-index', '--index-info'],
584 stdin=_subprocess.PIPE,
585 additional_env={'GIT_INDEX_FILE': path}) as git:
586 for line in notmuch.stdout:
587 if line.strip().startswith('#'):
589 (tags_string, id) = [_.strip() for _ in line.split(' -- id:')]
591 _unquote(tag[len(prefix):])
592 for tag in tags_string.split()
593 if tag.startswith(prefix)]
594 id = _xapian_unquote(string=id)
595 for line in _index_tags_for_message(
596 id=id, status='A', tags=tags):
597 git.stdin.write(line)
601 def _index_tags_for_message(id, status, tags):
603 Update the Git index to either create or delete an empty file.
605 Neither 'id' nor the tags in 'tags' should be encoded/escaped.
612 hash = '0000000000000000000000000000000000000000'
615 path = 'tags/{id}/{tag}'.format(
616 id=_hex_quote(string=id), tag=_hex_quote(string=tag))
617 yield '{mode} {hash}\t{path}\n'.format(mode=mode, hash=hash, path=path)
620 def _diff_index(index, filter):
622 Get an {id: {tag, ...}} dict for a given filter.
624 For example, use 'A' to find added tags, and 'D' to find deleted tags.
626 s = _collections.defaultdict(set)
629 'diff-index', '--cached', '--diff-filter', filter,
630 '--name-only', 'HEAD'],
631 additional_env={'GIT_INDEX_FILE': index},
632 stdout=_subprocess.PIPE) as p:
633 # Once we drop Python < 3.3, we can use 'yield from' here
634 for id, tag in _unpack_diff_lines(stream=p.stdout):
639 def _diff_refs(filter, a='HEAD', b='@{upstream}'):
641 args=['diff', '--diff-filter', filter, '--name-only', a, b],
642 stdout=_subprocess.PIPE) as p:
643 # Once we drop Python < 3.3, we can use 'yield from' here
644 for id, tag in _unpack_diff_lines(stream=p.stdout):
648 def _unpack_diff_lines(stream):
649 "Iterate through (id, tag) tuples in a diff stream."
651 match = _TAG_FILE_REGEX.match(line.strip())
653 message = 'non-tag line in diff: {!r}'.format(line.strip())
654 if line.startswith(_TAG_DIRECTORY):
655 raise ValueError(message)
658 id = _unquote(match.group('id'))
659 tag = _unquote(match.group('tag'))
663 def _help(parser, command=None):
665 Show help for an nmbug command.
667 Because some folks prefer:
673 $ nmbug COMMAND --help
676 parser.parse_args([command, '--help'])
678 parser.parse_args(['--help'])
681 if __name__ == '__main__':
684 parser = argparse.ArgumentParser(
685 description=__doc__.strip(),
686 formatter_class=argparse.RawDescriptionHelpFormatter)
688 '-C', '--git-dir', metavar='REPO',
689 help='Git repository to operate on.')
691 '-p', '--tag-prefix', metavar='PREFIX',
692 default = _os.getenv('NMBPREFIX', 'notmuch::'),
693 help='Prefix of tags to operate on.')
696 choices=['critical', 'error', 'warning', 'info', 'debug'],
697 help='Log verbosity. Defaults to {!r}.'.format(
698 _logging.getLevelName(_LOG.level).lower()))
700 help = _functools.partial(_help, parser=parser)
701 help.__doc__ = _help.__doc__
702 subparsers = parser.add_subparsers(
705 'For help on a particular command, run: '
706 "'%(prog)s ... <command> --help'."))
720 func = locals()[command]
721 doc = _textwrap.dedent(func.__doc__).strip().replace('%', '%%')
722 subparser = subparsers.add_parser(
724 help=doc.splitlines()[0],
726 formatter_class=argparse.RawDescriptionHelpFormatter)
727 subparser.set_defaults(func=func)
728 if command == 'archive':
729 subparser.add_argument(
730 'treeish', metavar='TREE-ISH', nargs='?', default='HEAD',
732 'The tree or commit to produce an archive for. Defaults '
734 subparser.add_argument(
735 'args', metavar='ARG', nargs='*',
737 "Argument passed through to 'git archive'. Set anything "
738 'before <tree-ish>, see git-archive(1) for details.'))
739 elif command == 'clone':
740 subparser.add_argument(
743 'The (possibly remote) repository to clone from. See the '
744 'URLS section of git-clone(1) for more information on '
745 'specifying repositories.'))
746 elif command == 'commit':
747 subparser.add_argument(
748 'message', metavar='MESSAGE', default='', nargs='?',
749 help='Text for the commit message.')
750 elif command == 'fetch':
751 subparser.add_argument(
752 'remote', metavar='REMOTE', nargs='?',
754 'Override the default configured in branch.<name>.remote '
755 'to fetch from a particular remote repository (e.g. '
757 elif command == 'help':
758 subparser.add_argument(
759 'command', metavar='COMMAND', nargs='?',
760 help='The command to show help for.')
761 elif command == 'log':
762 subparser.add_argument(
763 'args', metavar='ARG', nargs='*',
764 help="Additional argument passed through to 'git log'.")
765 elif command == 'merge':
766 subparser.add_argument(
767 'reference', metavar='REFERENCE', default='@{upstream}',
770 'Reference, usually other branch heads, to merge into '
771 "our branch. Defaults to '@{upstream}'."))
772 elif command == 'pull':
773 subparser.add_argument(
774 'repository', metavar='REPOSITORY', default=None, nargs='?',
776 'The "remote" repository that is the source of the pull. '
777 'This parameter can be either a URL (see the section GIT '
778 'URLS in git-pull(1)) or the name of a remote (see the '
779 'section REMOTES in git-pull(1)).'))
780 subparser.add_argument(
781 'refspecs', metavar='REFSPEC', default=None, nargs='*',
783 'Refspec (usually a branch name) to fetch and merge. See '
784 'the <refspec> entry in the OPTIONS section of '
785 'git-pull(1) for other possibilities.'))
786 elif command == 'push':
787 subparser.add_argument(
788 'repository', metavar='REPOSITORY', default=None, nargs='?',
790 'The "remote" repository that is the destination of the '
791 'push. This parameter can be either a URL (see the '
792 'section GIT URLS in git-push(1)) or the name of a remote '
793 '(see the section REMOTES in git-push(1)).'))
794 subparser.add_argument(
795 'refspecs', metavar='REFSPEC', default=None, nargs='*',
797 'Refspec (usually a branch name) to push. See '
798 'the <refspec> entry in the OPTIONS section of '
799 'git-push(1) for other possibilities.'))
801 args = parser.parse_args()
804 NMBGIT = args.git_dir
806 NMBGIT = _os.path.expanduser(
807 _os.getenv('NMBGIT', _os.path.join('~', '.nmbug')))
808 _NMBGIT = _os.path.join(NMBGIT, '.git')
809 if _os.path.isdir(_NMBGIT):
812 TAG_PREFIX = args.tag_prefix
813 _ENCODED_TAG_PREFIX = _hex_quote(TAG_PREFIX, safe='+@=,') # quote ':'
816 level = getattr(_logging, args.log_level.upper())
819 if not getattr(args, 'func', None):
823 if args.func == help:
824 arg_names = ['command']
826 (arg_names, varargs, varkw) = _inspect.getargs(args.func.__code__)
827 kwargs = {key: getattr(args, key) for key in arg_names if key in args}
830 except SubprocessError as e:
831 if _LOG.level == _logging.DEBUG:
832 raise # don't mask the traceback