3 # Copyright (c) 2011-2014 David Bremner <david@tethera.net>
4 # W. Trevor King <wking@tremily.us>
6 # This program is free software: you can redistribute it and/or modify
7 # it under the terms of the GNU General Public License as published by
8 # the Free Software Foundation, either version 3 of the License, or
9 # (at your option) any later version.
11 # This program is distributed in the hope that it will be useful,
12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 # GNU General Public License for more details.
16 # You should have received a copy of the GNU General Public License
17 # along with this program. If not, see https://www.gnu.org/licenses/ .
20 Manage notmuch tags with Git
23 from __future__ import print_function
24 from __future__ import unicode_literals
26 import codecs as _codecs
27 import collections as _collections
28 import functools as _functools
29 import inspect as _inspect
30 import locale as _locale
31 import logging as _logging
34 import shutil as _shutil
35 import subprocess as _subprocess
37 import tempfile as _tempfile
38 import textwrap as _textwrap
39 from urllib.parse import quote as _quote
40 from urllib.parse import unquote as _unquote
42 _LOG = _logging.getLogger('nmbug')
43 _LOG.setLevel(_logging.WARNING)
44 _LOG.addHandler(_logging.StreamHandler())
46 NOTMUCH_GIT_DIR = None
49 _HEX_ESCAPE_REGEX = _re.compile('%[0-9A-F]{2}')
50 _TAG_DIRECTORY = 'tags/'
51 _TAG_FILE_REGEX = _re.compile(_TAG_DIRECTORY + '(?P<id>[^/]*)/(?P<tag>[^/]*)')
53 # magic hash for Git (git hash-object -t blob /dev/null)
54 _EMPTYBLOB = 'e69de29bb2d1d6434b8b29ae775ad8c2e48c5391'
56 def _hex_quote(string, safe='+@=:,'):
58 quote('abc def') -> 'abc%20def'.
60 Wrap urllib.parse.quote with additional safe characters (in
61 addition to letters, digits, and '_.-') and lowercase hex digits
62 (e.g. '%3a' instead of '%3A').
64 uppercase_escapes = _quote(string, safe)
65 return _HEX_ESCAPE_REGEX.sub(
66 lambda match: match.group(0).lower(),
69 def _xapian_quote(string):
71 Quote a string for Xapian's QueryParser.
73 Xapian uses double-quotes for quoting strings. You can escape
74 internal quotes by repeating them [1,2,3].
76 [1]: https://trac.xapian.org/ticket/128#comment:2
77 [2]: https://trac.xapian.org/ticket/128#comment:17
78 [3]: https://trac.xapian.org/changeset/13823/svn
80 return '"{0}"'.format(string.replace('"', '""'))
83 def _xapian_unquote(string):
85 Unquote a Xapian-quoted string.
87 if string.startswith('"') and string.endswith('"'):
88 return string[1:-1].replace('""', '"')
92 class SubprocessError(RuntimeError):
93 "A subprocess exited with a nonzero status"
94 def __init__(self, args, status, stdout=None, stderr=None):
98 msg = '{args} exited with {status}'.format(args=args, status=status)
100 msg = '{msg}: {stderr}'.format(msg=msg, stderr=stderr)
101 super(SubprocessError, self).__init__(msg)
104 class _SubprocessContextManager(object):
106 PEP 343 context manager for subprocesses.
108 'expect' holds a tuple of acceptable exit codes, otherwise we'll
109 raise a SubprocessError in __exit__.
111 def __init__(self, process, args, expect=(0,)):
112 self._process = process
114 self._expect = expect
119 def __exit__(self, type, value, traceback):
120 for name in ['stdin', 'stdout', 'stderr']:
121 stream = getattr(self._process, name)
124 setattr(self._process, name, None)
125 status = self._process.wait()
127 'collect {args} with status {status} (expected {expect})'.format(
128 args=self._args, status=status, expect=self._expect))
129 if status not in self._expect:
130 raise SubprocessError(args=self._args, status=status)
133 return self._process.wait()
136 def _spawn(args, input=None, additional_env=None, wait=False, stdin=None,
137 stdout=None, stderr=None, encoding=_locale.getpreferredencoding(),
138 expect=(0,), **kwargs):
139 """Spawn a subprocess, and optionally wait for it to finish.
141 This wrapper around subprocess.Popen has two modes, depending on
142 the truthiness of 'wait'. If 'wait' is true, we use p.communicate
143 internally to write 'input' to the subprocess's stdin and read
144 from it's stdout/stderr. If 'wait' is False, we return a
145 _SubprocessContextManager instance for fancier handling
146 (e.g. piping between processes).
148 For 'wait' calls when you want to write to the subprocess's stdin,
149 you only need to set 'input' to your content. When 'input' is not
150 None but 'stdin' is, we'll automatically set 'stdin' to PIPE
151 before calling Popen. This avoids having the subprocess
152 accidentally inherit the launching process's stdin.
154 _LOG.debug('spawn {args} (additional env. var.: {env})'.format(
155 args=args, env=additional_env))
156 if not stdin and input is not None:
157 stdin = _subprocess.PIPE
159 if not kwargs.get('env'):
160 kwargs['env'] = dict(_os.environ)
161 kwargs['env'].update(additional_env)
162 p = _subprocess.Popen(
163 args, stdin=stdin, stdout=stdout, stderr=stderr, **kwargs)
165 if hasattr(input, 'encode'):
166 input = input.encode(encoding)
167 (stdout, stderr) = p.communicate(input=input)
170 'collect {args} with status {status} (expected {expect})'.format(
171 args=args, status=status, expect=expect))
172 if stdout is not None:
173 stdout = stdout.decode(encoding)
174 if stderr is not None:
175 stderr = stderr.decode(encoding)
176 if status not in expect:
177 raise SubprocessError(
178 args=args, status=status, stdout=stdout, stderr=stderr)
179 return (status, stdout, stderr)
180 if p.stdin and not stdin:
184 p.stdin = _codecs.getwriter(encoding=encoding)(stream=p.stdin)
185 stream_reader = _codecs.getreader(encoding=encoding)
187 p.stdout = stream_reader(stream=p.stdout)
189 p.stderr = stream_reader(stream=p.stderr)
190 return _SubprocessContextManager(args=args, process=p, expect=expect)
193 def _git(args, **kwargs):
194 args = ['git', '--git-dir', NOTMUCH_GIT_DIR] + list(args)
195 return _spawn(args=args, **kwargs)
198 def _get_current_branch():
199 """Get the name of the current branch.
201 Return 'None' if we're not on a branch.
204 (status, branch, stderr) = _git(
205 args=['symbolic-ref', '--short', 'HEAD'],
206 stdout=_subprocess.PIPE, stderr=_subprocess.PIPE, wait=True)
207 except SubprocessError as e:
208 if 'not a symbolic ref' in e:
211 return branch.strip()
215 "Get the default remote for the current branch."
216 local_branch = _get_current_branch()
217 (status, remote, stderr) = _git(
218 args=['config', 'branch.{0}.remote'.format(local_branch)],
219 stdout=_subprocess.PIPE, wait=True)
220 return remote.strip()
223 def get_tags(prefix=None):
224 "Get a list of tags with a given prefix."
227 (status, stdout, stderr) = _spawn(
228 args=['notmuch', 'search', '--output=tags', '*'],
229 stdout=_subprocess.PIPE, wait=True)
230 return [tag for tag in stdout.splitlines() if tag.startswith(prefix)]
233 def archive(treeish='HEAD', args=()):
235 Dump a tar archive of the current nmbug tag set.
239 Each tag $tag for message with Message-Id $id is written to
242 tags/encode($id)/encode($tag)
244 The encoding preserves alphanumerics, and the characters
245 "+-_@=.:," (not the quotes). All other octets are replaced with
246 '%' followed by a two digit hex number.
248 _git(args=['archive', treeish] + list(args), wait=True)
251 def clone(repository):
253 Create a local nmbug repository from a remote source.
255 This wraps 'git clone', adding some options to avoid creating a
256 working tree while preserving remote-tracking branches and
259 with _tempfile.TemporaryDirectory(prefix='nmbug-clone.') as workdir:
262 'git', 'clone', '--no-checkout', '--separate-git-dir', NOTMUCH_GIT_DIR,
263 repository, workdir],
265 _git(args=['config', '--unset', 'core.worktree'], wait=True, expect=(0, 5))
266 _git(args=['config', 'core.bare', 'true'], wait=True)
267 (status, stdout, stderr) = _git(args=['show-ref', '--verify',
269 'refs/remotes/origin/config'],
273 _git(args=['branch', 'config', 'origin/config'], wait=True)
274 existing_tags = get_tags()
277 'Not checking out to avoid clobbering existing tags: {}'.format(
278 ', '.join(existing_tags)))
283 def _is_committed(status):
284 return len(status['added']) + len(status['deleted']) == 0
287 def commit(treeish='HEAD', message=None):
289 Commit prefix-matching tags from the notmuch database to Git.
291 status = get_status()
293 if _is_committed(status=status):
294 _LOG.warning('Nothing to commit')
297 _git(args=['read-tree', '--empty'], wait=True)
298 _git(args=['read-tree', treeish], wait=True)
300 _update_index(status=status)
303 stdout=_subprocess.PIPE,
305 (_, parent, _) = _git(
306 args=['rev-parse', treeish],
307 stdout=_subprocess.PIPE,
309 (_, commit, _) = _git(
310 args=['commit-tree', tree.strip(), '-p', parent.strip()],
312 stdout=_subprocess.PIPE,
315 args=['update-ref', treeish, commit.strip()],
316 stdout=_subprocess.PIPE,
318 except Exception as e:
319 _git(args=['read-tree', '--empty'], wait=True)
320 _git(args=['read-tree', treeish], wait=True)
323 def _update_index(status):
325 args=['update-index', '--index-info'],
326 stdin=_subprocess.PIPE) as p:
327 for id, tags in status['deleted'].items():
328 for line in _index_tags_for_message(id=id, status='D', tags=tags):
330 for id, tags in status['added'].items():
331 for line in _index_tags_for_message(id=id, status='A', tags=tags):
335 def fetch(remote=None):
337 Fetch changes from the remote repository.
339 See 'merge' to bring those changes into notmuch.
344 _git(args=args, wait=True)
347 def init(remote=None):
349 Create an empty nmbug repository.
351 This wraps 'git init' with a few extra steps to support subsequent
352 status and commit commands.
354 _spawn(args=['git', '--git-dir', NOTMUCH_GIT_DIR, 'init', '--bare'], wait=True)
355 _git(args=['config', 'core.logallrefupdates', 'true'], wait=True)
356 # create an empty blob (e69de29bb2d1d6434b8b29ae775ad8c2e48c5391)
357 _git(args=['hash-object', '-w', '--stdin'], input='', wait=True)
360 'commit', '--allow-empty', '-m', 'Start a new nmbug repository'
362 additional_env={'GIT_WORK_TREE': NOTMUCH_GIT_DIR},
368 Update the notmuch database from Git.
370 This is mainly useful to discard your changes in notmuch relative
373 status = get_status()
375 args=['notmuch', 'tag', '--batch'], stdin=_subprocess.PIPE) as p:
376 for id, tags in status['added'].items():
377 p.stdin.write(_batch_line(action='-', id=id, tags=tags))
378 for id, tags in status['deleted'].items():
379 p.stdin.write(_batch_line(action='+', id=id, tags=tags))
382 def _batch_line(action, id, tags):
384 'notmuch tag --batch' line for adding/removing tags.
386 Set 'action' to '-' to remove a tag or '+' to add the tags to a
389 tag_string = ' '.join(
390 '{action}{prefix}{tag}'.format(
391 action=action, prefix=_ENCODED_TAG_PREFIX, tag=_hex_quote(tag))
393 line = '{tags} -- id:{id}\n'.format(
394 tags=tag_string, id=_xapian_quote(string=id))
398 def _insist_committed():
399 "Die if the the notmuch tags don't match the current HEAD."
400 status = get_status()
401 if not _is_committed(status=status):
402 _LOG.error('\n'.join([
403 'Uncommitted changes to {prefix}* tags in notmuch',
405 "For a summary of changes, run 'nmbug status'",
406 "To save your changes, run 'nmbug commit' before merging/pull",
407 "To discard your changes, run 'nmbug checkout'",
408 ]).format(prefix=TAG_PREFIX))
412 def pull(repository=None, refspecs=None):
414 Pull (merge) remote repository changes to notmuch.
416 'pull' is equivalent to 'fetch' followed by 'merge'. We use the
417 Git-configured repository for your current branch
418 (branch.<name>.repository, likely 'origin', and
419 branch.<name>.merge, likely 'master').
422 if refspecs and not repository:
423 repository = _get_remote()
426 args.append(repository)
428 args.extend(refspecs)
429 with _tempfile.TemporaryDirectory(prefix='nmbug-pull.') as workdir:
435 additional_env={'GIT_WORK_TREE': workdir},
440 def merge(reference='@{upstream}'):
442 Merge changes from 'reference' into HEAD and load the result into notmuch.
444 The default reference is '@{upstream}'.
447 with _tempfile.TemporaryDirectory(prefix='nmbug-merge.') as workdir:
450 ['merge', reference]]:
453 additional_env={'GIT_WORK_TREE': workdir},
460 A simple wrapper for 'git log'.
462 After running 'nmbug fetch', you can inspect the changes with
463 'nmbug log HEAD..@{upstream}'.
465 # we don't want output trapping here, because we want the pager.
466 args = ['log', '--name-status', '--no-renames'] + list(args)
467 with _git(args=args, expect=(0, 1, -13)) as p:
471 def push(repository=None, refspecs=None):
472 "Push the local nmbug Git state to a remote repository."
473 if refspecs and not repository:
474 repository = _get_remote()
477 args.append(repository)
479 args.extend(refspecs)
480 _git(args=args, wait=True)
485 Show pending updates in notmuch or git repo.
487 Prints lines of the form
491 where n is a single character representing notmuch database status
495 Tag is present in notmuch database, but not committed to nmbug
496 (equivalently, tag has been deleted in nmbug repo, e.g. by a
497 pull, but not restored to notmuch database).
501 Tag is present in nmbug repo, but not restored to notmuch
502 database (equivalently, tag has been deleted in notmuch).
506 Message is unknown (missing from local notmuch database).
508 The second character (if present) represents a difference between
509 local and upstream branches. Typically 'nmbug fetch' needs to be
514 Tag is present in upstream, but not in the local Git branch.
518 Tag is present in local Git branch, but not upstream.
520 status = get_status()
521 # 'output' is a nested defaultdict for message status:
522 # * The outer dict is keyed by message id.
523 # * The inner dict is keyed by tag name.
524 # * The inner dict values are status strings (' a', 'Dd', ...).
525 output = _collections.defaultdict(
526 lambda : _collections.defaultdict(lambda : ' '))
527 for id, tags in status['added'].items():
529 output[id][tag] = 'A'
530 for id, tags in status['deleted'].items():
532 output[id][tag] = 'D'
533 for id, tags in status['missing'].items():
535 output[id][tag] = 'U'
537 for id, tag in _diff_refs(filter='A'):
538 output[id][tag] += 'a'
539 for id, tag in _diff_refs(filter='D'):
540 output[id][tag] += 'd'
541 for id, tag_status in sorted(output.items()):
542 for tag, status in sorted(tag_status.items()):
543 print('{status}\t{id}\t{tag}'.format(
544 status=status, id=id, tag=tag))
547 def _is_unmerged(ref='@{upstream}'):
549 (status, fetch_head, stderr) = _git(
550 args=['rev-parse', ref],
551 stdout=_subprocess.PIPE, stderr=_subprocess.PIPE, wait=True)
552 except SubprocessError as e:
553 if 'No upstream configured' in e.stderr:
556 (status, base, stderr) = _git(
557 args=['merge-base', 'HEAD', ref],
558 stdout=_subprocess.PIPE, wait=True)
559 return base != fetch_head
567 index = _index_tags()
568 maybe_deleted = _diff_index(index=index, filter='D')
569 for id, tags in maybe_deleted.items():
570 (_, stdout, stderr) = _spawn(
571 args=['notmuch', 'search', '--output=files', 'id:{0}'.format(id)],
572 stdout=_subprocess.PIPE,
575 status['deleted'][id] = tags
577 status['missing'][id] = tags
578 status['added'] = _diff_index(index=index, filter='A')
584 "Write notmuch tags to the nmbug.index."
585 path = _os.path.join(NOTMUCH_GIT_DIR, 'nmbug.index')
586 query = ' '.join('tag:"{tag}"'.format(tag=tag) for tag in get_tags())
587 prefix = '+{0}'.format(_ENCODED_TAG_PREFIX)
589 args=['read-tree', '--empty'],
590 additional_env={'GIT_INDEX_FILE': path}, wait=True)
592 args=['notmuch', 'dump', '--format=batch-tag', '--', query],
593 stdout=_subprocess.PIPE) as notmuch:
595 args=['update-index', '--index-info'],
596 stdin=_subprocess.PIPE,
597 additional_env={'GIT_INDEX_FILE': path}) as git:
598 for line in notmuch.stdout:
599 if line.strip().startswith('#'):
601 (tags_string, id) = [_.strip() for _ in line.split(' -- id:')]
603 _unquote(tag[len(prefix):])
604 for tag in tags_string.split()
605 if tag.startswith(prefix)]
606 id = _xapian_unquote(string=id)
607 for line in _index_tags_for_message(
608 id=id, status='A', tags=tags):
609 git.stdin.write(line)
613 def _index_tags_for_message(id, status, tags):
615 Update the Git index to either create or delete an empty file.
617 Neither 'id' nor the tags in 'tags' should be encoded/escaped.
624 hash = '0000000000000000000000000000000000000000'
627 path = 'tags/{id}/{tag}'.format(
628 id=_hex_quote(string=id), tag=_hex_quote(string=tag))
629 yield '{mode} {hash}\t{path}\n'.format(mode=mode, hash=hash, path=path)
632 def _diff_index(index, filter):
634 Get an {id: {tag, ...}} dict for a given filter.
636 For example, use 'A' to find added tags, and 'D' to find deleted tags.
638 s = _collections.defaultdict(set)
641 'diff-index', '--cached', '--diff-filter', filter,
642 '--name-only', 'HEAD'],
643 additional_env={'GIT_INDEX_FILE': index},
644 stdout=_subprocess.PIPE) as p:
645 # Once we drop Python < 3.3, we can use 'yield from' here
646 for id, tag in _unpack_diff_lines(stream=p.stdout):
651 def _diff_refs(filter, a='HEAD', b='@{upstream}'):
653 args=['diff', '--diff-filter', filter, '--name-only', a, b],
654 stdout=_subprocess.PIPE) as p:
655 # Once we drop Python < 3.3, we can use 'yield from' here
656 for id, tag in _unpack_diff_lines(stream=p.stdout):
660 def _unpack_diff_lines(stream):
661 "Iterate through (id, tag) tuples in a diff stream."
663 match = _TAG_FILE_REGEX.match(line.strip())
665 message = 'non-tag line in diff: {!r}'.format(line.strip())
666 if line.startswith(_TAG_DIRECTORY):
667 raise ValueError(message)
670 id = _unquote(match.group('id'))
671 tag = _unquote(match.group('tag'))
675 def _help(parser, command=None):
677 Show help for an nmbug command.
679 Because some folks prefer:
685 $ nmbug COMMAND --help
688 parser.parse_args([command, '--help'])
690 parser.parse_args(['--help'])
693 if __name__ == '__main__':
696 parser = argparse.ArgumentParser(
697 description=__doc__.strip(),
698 formatter_class=argparse.RawDescriptionHelpFormatter)
700 '-C', '--git-dir', metavar='REPO',
701 help='Git repository to operate on.')
703 '-p', '--tag-prefix', metavar='PREFIX',
704 default = _os.getenv('NOTMUCH_GIT_PREFIX', 'notmuch::'),
705 help='Prefix of tags to operate on.')
708 choices=['critical', 'error', 'warning', 'info', 'debug'],
709 help='Log verbosity. Defaults to {!r}.'.format(
710 _logging.getLevelName(_LOG.level).lower()))
712 help = _functools.partial(_help, parser=parser)
713 help.__doc__ = _help.__doc__
714 subparsers = parser.add_subparsers(
717 'For help on a particular command, run: '
718 "'%(prog)s ... <command> --help'."))
733 func = locals()[command]
734 doc = _textwrap.dedent(func.__doc__).strip().replace('%', '%%')
735 subparser = subparsers.add_parser(
737 help=doc.splitlines()[0],
739 formatter_class=argparse.RawDescriptionHelpFormatter)
740 subparser.set_defaults(func=func)
741 if command == 'archive':
742 subparser.add_argument(
743 'treeish', metavar='TREE-ISH', nargs='?', default='HEAD',
745 'The tree or commit to produce an archive for. Defaults '
747 subparser.add_argument(
748 'args', metavar='ARG', nargs='*',
750 "Argument passed through to 'git archive'. Set anything "
751 'before <tree-ish>, see git-archive(1) for details.'))
752 elif command == 'clone':
753 subparser.add_argument(
756 'The (possibly remote) repository to clone from. See the '
757 'URLS section of git-clone(1) for more information on '
758 'specifying repositories.'))
759 elif command == 'commit':
760 subparser.add_argument(
761 'message', metavar='MESSAGE', default='', nargs='?',
762 help='Text for the commit message.')
763 elif command == 'fetch':
764 subparser.add_argument(
765 'remote', metavar='REMOTE', nargs='?',
767 'Override the default configured in branch.<name>.remote '
768 'to fetch from a particular remote repository (e.g. '
770 elif command == 'help':
771 subparser.add_argument(
772 'command', metavar='COMMAND', nargs='?',
773 help='The command to show help for.')
774 elif command == 'log':
775 subparser.add_argument(
776 'args', metavar='ARG', nargs='*',
777 help="Additional argument passed through to 'git log'.")
778 elif command == 'merge':
779 subparser.add_argument(
780 'reference', metavar='REFERENCE', default='@{upstream}',
783 'Reference, usually other branch heads, to merge into '
784 "our branch. Defaults to '@{upstream}'."))
785 elif command == 'pull':
786 subparser.add_argument(
787 'repository', metavar='REPOSITORY', default=None, nargs='?',
789 'The "remote" repository that is the source of the pull. '
790 'This parameter can be either a URL (see the section GIT '
791 'URLS in git-pull(1)) or the name of a remote (see the '
792 'section REMOTES in git-pull(1)).'))
793 subparser.add_argument(
794 'refspecs', metavar='REFSPEC', default=None, nargs='*',
796 'Refspec (usually a branch name) to fetch and merge. See '
797 'the <refspec> entry in the OPTIONS section of '
798 'git-pull(1) for other possibilities.'))
799 elif command == 'push':
800 subparser.add_argument(
801 'repository', metavar='REPOSITORY', default=None, nargs='?',
803 'The "remote" repository that is the destination of the '
804 'push. This parameter can be either a URL (see the '
805 'section GIT URLS in git-push(1)) or the name of a remote '
806 '(see the section REMOTES in git-push(1)).'))
807 subparser.add_argument(
808 'refspecs', metavar='REFSPEC', default=None, nargs='*',
810 'Refspec (usually a branch name) to push. See '
811 'the <refspec> entry in the OPTIONS section of '
812 'git-push(1) for other possibilities.'))
814 args = parser.parse_args()
817 NOTMUCH_GIT_DIR = args.git_dir
819 NOTMUCH_GIT_DIR = _os.path.expanduser(
820 _os.getenv('NOTMUCH_GIT_DIR', _os.path.join('~', '.nmbug')))
821 _NOTMUCH_GIT_DIR = _os.path.join(NOTMUCH_GIT_DIR, '.git')
822 if _os.path.isdir(_NOTMUCH_GIT_DIR):
823 NOTMUCH_GIT_DIR = _NOTMUCH_GIT_DIR
825 TAG_PREFIX = args.tag_prefix
826 _ENCODED_TAG_PREFIX = _hex_quote(TAG_PREFIX, safe='+@=,') # quote ':'
829 level = getattr(_logging, args.log_level.upper())
833 for var in ['NOTMUCH_GIT_DIR', 'NOTMUCH_GIT_PREFIX', 'NOTMUCH_PROFILE', 'NOTMUCH_CONFIG' ]:
834 _LOG.debug('env {:s} = {:s}'.format(var, _os.getenv(var,'%None%')))
836 if not getattr(args, 'func', None):
840 if args.func == help:
841 arg_names = ['command']
843 (arg_names, varargs, varkw) = _inspect.getargs(args.func.__code__)
844 kwargs = {key: getattr(args, key) for key in arg_names if key in args}
847 except SubprocessError as e:
848 if _LOG.level == _logging.DEBUG:
849 raise # don't mask the traceback