3 # Copyright (c) 2011-2014 David Bremner <david@tethera.net>
4 # W. Trevor King <wking@tremily.us>
6 # This program is free software: you can redistribute it and/or modify
7 # it under the terms of the GNU General Public License as published by
8 # the Free Software Foundation, either version 3 of the License, or
9 # (at your option) any later version.
11 # This program is distributed in the hope that it will be useful,
12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 # GNU General Public License for more details.
16 # You should have received a copy of the GNU General Public License
17 # along with this program. If not, see https://www.gnu.org/licenses/ .
20 Manage notmuch tags with Git
22 Environment variables:
24 * NMBGIT specifies the location of the git repository used by nmbug.
25 If not specified $HOME/.nmbug is used.
26 * NMBPREFIX specifies the prefix in the notmuch database for tags of
27 interest to nmbug. If not specified 'notmuch::' is used.
30 from __future__ import print_function
31 from __future__ import unicode_literals
33 import codecs as _codecs
34 import collections as _collections
35 import functools as _functools
36 import inspect as _inspect
37 import locale as _locale
38 import logging as _logging
41 import shutil as _shutil
42 import subprocess as _subprocess
44 import tempfile as _tempfile
45 import textwrap as _textwrap
46 from urllib.parse import quote as _quote
47 from urllib.parse import unquote as _unquote
49 _LOG = _logging.getLogger('nmbug')
50 _LOG.setLevel(_logging.WARNING)
51 _LOG.addHandler(_logging.StreamHandler())
53 NMBGIT = _os.path.expanduser(
54 _os.getenv('NMBGIT', _os.path.join('~', '.nmbug')))
55 _NMBGIT = _os.path.join(NMBGIT, '.git')
56 if _os.path.isdir(_NMBGIT):
59 TAG_PREFIX = _os.getenv('NMBPREFIX', 'notmuch::')
60 _HEX_ESCAPE_REGEX = _re.compile('%[0-9A-F]{2}')
61 _TAG_DIRECTORY = 'tags/'
62 _TAG_FILE_REGEX = _re.compile(_TAG_DIRECTORY + '(?P<id>[^/]*)/(?P<tag>[^/]*)')
64 # magic hash for Git (git hash-object -t blob /dev/null)
65 _EMPTYBLOB = 'e69de29bb2d1d6434b8b29ae775ad8c2e48c5391'
67 def _hex_quote(string, safe='+@=:,'):
69 quote('abc def') -> 'abc%20def'.
71 Wrap urllib.parse.quote with additional safe characters (in
72 addition to letters, digits, and '_.-') and lowercase hex digits
73 (e.g. '%3a' instead of '%3A').
75 uppercase_escapes = _quote(string, safe)
76 return _HEX_ESCAPE_REGEX.sub(
77 lambda match: match.group(0).lower(),
81 _ENCODED_TAG_PREFIX = _hex_quote(TAG_PREFIX, safe='+@=,') # quote ':'
84 def _xapian_quote(string):
86 Quote a string for Xapian's QueryParser.
88 Xapian uses double-quotes for quoting strings. You can escape
89 internal quotes by repeating them [1,2,3].
91 [1]: https://trac.xapian.org/ticket/128#comment:2
92 [2]: https://trac.xapian.org/ticket/128#comment:17
93 [3]: https://trac.xapian.org/changeset/13823/svn
95 return '"{0}"'.format(string.replace('"', '""'))
98 def _xapian_unquote(string):
100 Unquote a Xapian-quoted string.
102 if string.startswith('"') and string.endswith('"'):
103 return string[1:-1].replace('""', '"')
107 class SubprocessError(RuntimeError):
108 "A subprocess exited with a nonzero status"
109 def __init__(self, args, status, stdout=None, stderr=None):
113 msg = '{args} exited with {status}'.format(args=args, status=status)
115 msg = '{msg}: {stderr}'.format(msg=msg, stderr=stderr)
116 super(SubprocessError, self).__init__(msg)
119 class _SubprocessContextManager(object):
121 PEP 343 context manager for subprocesses.
123 'expect' holds a tuple of acceptable exit codes, otherwise we'll
124 raise a SubprocessError in __exit__.
126 def __init__(self, process, args, expect=(0,)):
127 self._process = process
129 self._expect = expect
134 def __exit__(self, type, value, traceback):
135 for name in ['stdin', 'stdout', 'stderr']:
136 stream = getattr(self._process, name)
139 setattr(self._process, name, None)
140 status = self._process.wait()
142 'collect {args} with status {status} (expected {expect})'.format(
143 args=self._args, status=status, expect=self._expect))
144 if status not in self._expect:
145 raise SubprocessError(args=self._args, status=status)
148 return self._process.wait()
151 def _spawn(args, input=None, additional_env=None, wait=False, stdin=None,
152 stdout=None, stderr=None, encoding=_locale.getpreferredencoding(),
153 expect=(0,), **kwargs):
154 """Spawn a subprocess, and optionally wait for it to finish.
156 This wrapper around subprocess.Popen has two modes, depending on
157 the truthiness of 'wait'. If 'wait' is true, we use p.communicate
158 internally to write 'input' to the subprocess's stdin and read
159 from it's stdout/stderr. If 'wait' is False, we return a
160 _SubprocessContextManager instance for fancier handling
161 (e.g. piping between processes).
163 For 'wait' calls when you want to write to the subprocess's stdin,
164 you only need to set 'input' to your content. When 'input' is not
165 None but 'stdin' is, we'll automatically set 'stdin' to PIPE
166 before calling Popen. This avoids having the subprocess
167 accidentally inherit the launching process's stdin.
169 _LOG.debug('spawn {args} (additional env. var.: {env})'.format(
170 args=args, env=additional_env))
171 if not stdin and input is not None:
172 stdin = _subprocess.PIPE
174 if not kwargs.get('env'):
175 kwargs['env'] = dict(_os.environ)
176 kwargs['env'].update(additional_env)
177 p = _subprocess.Popen(
178 args, stdin=stdin, stdout=stdout, stderr=stderr, **kwargs)
180 if hasattr(input, 'encode'):
181 input = input.encode(encoding)
182 (stdout, stderr) = p.communicate(input=input)
185 'collect {args} with status {status} (expected {expect})'.format(
186 args=args, status=status, expect=expect))
187 if stdout is not None:
188 stdout = stdout.decode(encoding)
189 if stderr is not None:
190 stderr = stderr.decode(encoding)
191 if status not in expect:
192 raise SubprocessError(
193 args=args, status=status, stdout=stdout, stderr=stderr)
194 return (status, stdout, stderr)
195 if p.stdin and not stdin:
199 p.stdin = _codecs.getwriter(encoding=encoding)(stream=p.stdin)
200 stream_reader = _codecs.getreader(encoding=encoding)
202 p.stdout = stream_reader(stream=p.stdout)
204 p.stderr = stream_reader(stream=p.stderr)
205 return _SubprocessContextManager(args=args, process=p, expect=expect)
208 def _git(args, **kwargs):
209 args = ['git', '--git-dir', NMBGIT] + list(args)
210 return _spawn(args=args, **kwargs)
213 def _get_current_branch():
214 """Get the name of the current branch.
216 Return 'None' if we're not on a branch.
219 (status, branch, stderr) = _git(
220 args=['symbolic-ref', '--short', 'HEAD'],
221 stdout=_subprocess.PIPE, stderr=_subprocess.PIPE, wait=True)
222 except SubprocessError as e:
223 if 'not a symbolic ref' in e:
226 return branch.strip()
230 "Get the default remote for the current branch."
231 local_branch = _get_current_branch()
232 (status, remote, stderr) = _git(
233 args=['config', 'branch.{0}.remote'.format(local_branch)],
234 stdout=_subprocess.PIPE, wait=True)
235 return remote.strip()
238 def get_tags(prefix=None):
239 "Get a list of tags with a given prefix."
242 (status, stdout, stderr) = _spawn(
243 args=['notmuch', 'search', '--output=tags', '*'],
244 stdout=_subprocess.PIPE, wait=True)
245 return [tag for tag in stdout.splitlines() if tag.startswith(prefix)]
248 def archive(treeish='HEAD', args=()):
250 Dump a tar archive of the current nmbug tag set.
254 Each tag $tag for message with Message-Id $id is written to
257 tags/encode($id)/encode($tag)
259 The encoding preserves alphanumerics, and the characters
260 "+-_@=.:," (not the quotes). All other octets are replaced with
261 '%' followed by a two digit hex number.
263 _git(args=['archive', treeish] + list(args), wait=True)
266 def clone(repository):
268 Create a local nmbug repository from a remote source.
270 This wraps 'git clone', adding some options to avoid creating a
271 working tree while preserving remote-tracking branches and
274 with _tempfile.TemporaryDirectory(prefix='nmbug-clone.') as workdir:
277 'git', 'clone', '--no-checkout', '--separate-git-dir', NMBGIT,
278 repository, workdir],
280 _git(args=['config', '--unset', 'core.worktree'], wait=True, expect=(0, 5))
281 _git(args=['config', 'core.bare', 'true'], wait=True)
282 _git(args=['branch', 'config', 'origin/config'], wait=True)
283 existing_tags = get_tags()
286 'Not checking out to avoid clobbering existing tags: {}'.format(
287 ', '.join(existing_tags)))
292 def _is_committed(status):
293 return len(status['added']) + len(status['deleted']) == 0
296 def commit(treeish='HEAD', message=None):
298 Commit prefix-matching tags from the notmuch database to Git.
300 status = get_status()
302 if _is_committed(status=status):
303 _LOG.warning('Nothing to commit')
306 _git(args=['read-tree', '--empty'], wait=True)
307 _git(args=['read-tree', treeish], wait=True)
309 _update_index(status=status)
312 stdout=_subprocess.PIPE,
314 (_, parent, _) = _git(
315 args=['rev-parse', treeish],
316 stdout=_subprocess.PIPE,
318 (_, commit, _) = _git(
319 args=['commit-tree', tree.strip(), '-p', parent.strip()],
321 stdout=_subprocess.PIPE,
324 args=['update-ref', treeish, commit.strip()],
325 stdout=_subprocess.PIPE,
327 except Exception as e:
328 _git(args=['read-tree', '--empty'], wait=True)
329 _git(args=['read-tree', treeish], wait=True)
332 def _update_index(status):
334 args=['update-index', '--index-info'],
335 stdin=_subprocess.PIPE) as p:
336 for id, tags in status['deleted'].items():
337 for line in _index_tags_for_message(id=id, status='D', tags=tags):
339 for id, tags in status['added'].items():
340 for line in _index_tags_for_message(id=id, status='A', tags=tags):
344 def fetch(remote=None):
346 Fetch changes from the remote repository.
348 See 'merge' to bring those changes into notmuch.
353 _git(args=args, wait=True)
358 Update the notmuch database from Git.
360 This is mainly useful to discard your changes in notmuch relative
363 status = get_status()
365 args=['notmuch', 'tag', '--batch'], stdin=_subprocess.PIPE) as p:
366 for id, tags in status['added'].items():
367 p.stdin.write(_batch_line(action='-', id=id, tags=tags))
368 for id, tags in status['deleted'].items():
369 p.stdin.write(_batch_line(action='+', id=id, tags=tags))
372 def _batch_line(action, id, tags):
374 'notmuch tag --batch' line for adding/removing tags.
376 Set 'action' to '-' to remove a tag or '+' to add the tags to a
379 tag_string = ' '.join(
380 '{action}{prefix}{tag}'.format(
381 action=action, prefix=_ENCODED_TAG_PREFIX, tag=_hex_quote(tag))
383 line = '{tags} -- id:{id}\n'.format(
384 tags=tag_string, id=_xapian_quote(string=id))
388 def _insist_committed():
389 "Die if the the notmuch tags don't match the current HEAD."
390 status = get_status()
391 if not _is_committed(status=status):
392 _LOG.error('\n'.join([
393 'Uncommitted changes to {prefix}* tags in notmuch',
395 "For a summary of changes, run 'nmbug status'",
396 "To save your changes, run 'nmbug commit' before merging/pull",
397 "To discard your changes, run 'nmbug checkout'",
398 ]).format(prefix=TAG_PREFIX))
402 def pull(repository=None, refspecs=None):
404 Pull (merge) remote repository changes to notmuch.
406 'pull' is equivalent to 'fetch' followed by 'merge'. We use the
407 Git-configured repository for your current branch
408 (branch.<name>.repository, likely 'origin', and
409 branch.<name>.merge, likely 'master').
412 if refspecs and not repository:
413 repository = _get_remote()
416 args.append(repository)
418 args.extend(refspecs)
419 with _tempfile.TemporaryDirectory(prefix='nmbug-pull.') as workdir:
425 additional_env={'GIT_WORK_TREE': workdir},
430 def merge(reference='@{upstream}'):
432 Merge changes from 'reference' into HEAD and load the result into notmuch.
434 The default reference is '@{upstream}'.
437 with _tempfile.TemporaryDirectory(prefix='nmbug-merge.') as workdir:
440 ['merge', reference]]:
443 additional_env={'GIT_WORK_TREE': workdir},
450 A simple wrapper for 'git log'.
452 After running 'nmbug fetch', you can inspect the changes with
453 'nmbug log HEAD..@{upstream}'.
455 # we don't want output trapping here, because we want the pager.
456 args = ['log', '--name-status', '--no-renames'] + list(args)
457 with _git(args=args, expect=(0, 1, -13)) as p:
461 def push(repository=None, refspecs=None):
462 "Push the local nmbug Git state to a remote repository."
463 if refspecs and not repository:
464 repository = _get_remote()
467 args.append(repository)
469 args.extend(refspecs)
470 _git(args=args, wait=True)
475 Show pending updates in notmuch or git repo.
477 Prints lines of the form
481 where n is a single character representing notmuch database status
485 Tag is present in notmuch database, but not committed to nmbug
486 (equivalently, tag has been deleted in nmbug repo, e.g. by a
487 pull, but not restored to notmuch database).
491 Tag is present in nmbug repo, but not restored to notmuch
492 database (equivalently, tag has been deleted in notmuch).
496 Message is unknown (missing from local notmuch database).
498 The second character (if present) represents a difference between
499 local and upstream branches. Typically 'nmbug fetch' needs to be
504 Tag is present in upstream, but not in the local Git branch.
508 Tag is present in local Git branch, but not upstream.
510 status = get_status()
511 # 'output' is a nested defaultdict for message status:
512 # * The outer dict is keyed by message id.
513 # * The inner dict is keyed by tag name.
514 # * The inner dict values are status strings (' a', 'Dd', ...).
515 output = _collections.defaultdict(
516 lambda : _collections.defaultdict(lambda : ' '))
517 for id, tags in status['added'].items():
519 output[id][tag] = 'A'
520 for id, tags in status['deleted'].items():
522 output[id][tag] = 'D'
523 for id, tags in status['missing'].items():
525 output[id][tag] = 'U'
527 for id, tag in _diff_refs(filter='A'):
528 output[id][tag] += 'a'
529 for id, tag in _diff_refs(filter='D'):
530 output[id][tag] += 'd'
531 for id, tag_status in sorted(output.items()):
532 for tag, status in sorted(tag_status.items()):
533 print('{status}\t{id}\t{tag}'.format(
534 status=status, id=id, tag=tag))
537 def _is_unmerged(ref='@{upstream}'):
539 (status, fetch_head, stderr) = _git(
540 args=['rev-parse', ref],
541 stdout=_subprocess.PIPE, stderr=_subprocess.PIPE, wait=True)
542 except SubprocessError as e:
543 if 'No upstream configured' in e.stderr:
546 (status, base, stderr) = _git(
547 args=['merge-base', 'HEAD', ref],
548 stdout=_subprocess.PIPE, wait=True)
549 return base != fetch_head
557 index = _index_tags()
558 maybe_deleted = _diff_index(index=index, filter='D')
559 for id, tags in maybe_deleted.items():
560 (_, stdout, stderr) = _spawn(
561 args=['notmuch', 'search', '--output=files', 'id:{0}'.format(id)],
562 stdout=_subprocess.PIPE,
565 status['deleted'][id] = tags
567 status['missing'][id] = tags
568 status['added'] = _diff_index(index=index, filter='A')
574 "Write notmuch tags to the nmbug.index."
575 path = _os.path.join(NMBGIT, 'nmbug.index')
576 query = ' '.join('tag:"{tag}"'.format(tag=tag) for tag in get_tags())
577 prefix = '+{0}'.format(_ENCODED_TAG_PREFIX)
579 args=['read-tree', '--empty'],
580 additional_env={'GIT_INDEX_FILE': path}, wait=True)
582 args=['notmuch', 'dump', '--format=batch-tag', '--', query],
583 stdout=_subprocess.PIPE) as notmuch:
585 args=['update-index', '--index-info'],
586 stdin=_subprocess.PIPE,
587 additional_env={'GIT_INDEX_FILE': path}) as git:
588 for line in notmuch.stdout:
589 if line.strip().startswith('#'):
591 (tags_string, id) = [_.strip() for _ in line.split(' -- id:')]
593 _unquote(tag[len(prefix):])
594 for tag in tags_string.split()
595 if tag.startswith(prefix)]
596 id = _xapian_unquote(string=id)
597 for line in _index_tags_for_message(
598 id=id, status='A', tags=tags):
599 git.stdin.write(line)
603 def _index_tags_for_message(id, status, tags):
605 Update the Git index to either create or delete an empty file.
607 Neither 'id' nor the tags in 'tags' should be encoded/escaped.
614 hash = '0000000000000000000000000000000000000000'
617 path = 'tags/{id}/{tag}'.format(
618 id=_hex_quote(string=id), tag=_hex_quote(string=tag))
619 yield '{mode} {hash}\t{path}\n'.format(mode=mode, hash=hash, path=path)
622 def _diff_index(index, filter):
624 Get an {id: {tag, ...}} dict for a given filter.
626 For example, use 'A' to find added tags, and 'D' to find deleted tags.
628 s = _collections.defaultdict(set)
631 'diff-index', '--cached', '--diff-filter', filter,
632 '--name-only', 'HEAD'],
633 additional_env={'GIT_INDEX_FILE': index},
634 stdout=_subprocess.PIPE) as p:
635 # Once we drop Python < 3.3, we can use 'yield from' here
636 for id, tag in _unpack_diff_lines(stream=p.stdout):
641 def _diff_refs(filter, a='HEAD', b='@{upstream}'):
643 args=['diff', '--diff-filter', filter, '--name-only', a, b],
644 stdout=_subprocess.PIPE) as p:
645 # Once we drop Python < 3.3, we can use 'yield from' here
646 for id, tag in _unpack_diff_lines(stream=p.stdout):
650 def _unpack_diff_lines(stream):
651 "Iterate through (id, tag) tuples in a diff stream."
653 match = _TAG_FILE_REGEX.match(line.strip())
655 message = 'non-tag line in diff: {!r}'.format(line.strip())
656 if line.startswith(_TAG_DIRECTORY):
657 raise ValueError(message)
660 id = _unquote(match.group('id'))
661 tag = _unquote(match.group('tag'))
665 def _help(parser, command=None):
667 Show help for an nmbug command.
669 Because some folks prefer:
675 $ nmbug COMMAND --help
678 parser.parse_args([command, '--help'])
680 parser.parse_args(['--help'])
683 if __name__ == '__main__':
686 parser = argparse.ArgumentParser(
687 description=__doc__.strip(),
688 formatter_class=argparse.RawDescriptionHelpFormatter)
691 choices=['critical', 'error', 'warning', 'info', 'debug'],
692 help='Log verbosity. Defaults to {!r}.'.format(
693 _logging.getLevelName(_LOG.level).lower()))
695 help = _functools.partial(_help, parser=parser)
696 help.__doc__ = _help.__doc__
697 subparsers = parser.add_subparsers(
700 'For help on a particular command, run: '
701 "'%(prog)s ... <command> --help'."))
715 func = locals()[command]
716 doc = _textwrap.dedent(func.__doc__).strip().replace('%', '%%')
717 subparser = subparsers.add_parser(
719 help=doc.splitlines()[0],
721 formatter_class=argparse.RawDescriptionHelpFormatter)
722 subparser.set_defaults(func=func)
723 if command == 'archive':
724 subparser.add_argument(
725 'treeish', metavar='TREE-ISH', nargs='?', default='HEAD',
727 'The tree or commit to produce an archive for. Defaults '
729 subparser.add_argument(
730 'args', metavar='ARG', nargs='*',
732 "Argument passed through to 'git archive'. Set anything "
733 'before <tree-ish>, see git-archive(1) for details.'))
734 elif command == 'clone':
735 subparser.add_argument(
738 'The (possibly remote) repository to clone from. See the '
739 'URLS section of git-clone(1) for more information on '
740 'specifying repositories.'))
741 elif command == 'commit':
742 subparser.add_argument(
743 'message', metavar='MESSAGE', default='', nargs='?',
744 help='Text for the commit message.')
745 elif command == 'fetch':
746 subparser.add_argument(
747 'remote', metavar='REMOTE', nargs='?',
749 'Override the default configured in branch.<name>.remote '
750 'to fetch from a particular remote repository (e.g. '
752 elif command == 'help':
753 subparser.add_argument(
754 'command', metavar='COMMAND', nargs='?',
755 help='The command to show help for.')
756 elif command == 'log':
757 subparser.add_argument(
758 'args', metavar='ARG', nargs='*',
759 help="Additional argument passed through to 'git log'.")
760 elif command == 'merge':
761 subparser.add_argument(
762 'reference', metavar='REFERENCE', default='@{upstream}',
765 'Reference, usually other branch heads, to merge into '
766 "our branch. Defaults to '@{upstream}'."))
767 elif command == 'pull':
768 subparser.add_argument(
769 'repository', metavar='REPOSITORY', default=None, nargs='?',
771 'The "remote" repository that is the source of the pull. '
772 'This parameter can be either a URL (see the section GIT '
773 'URLS in git-pull(1)) or the name of a remote (see the '
774 'section REMOTES in git-pull(1)).'))
775 subparser.add_argument(
776 'refspecs', metavar='REFSPEC', default=None, nargs='*',
778 'Refspec (usually a branch name) to fetch and merge. See '
779 'the <refspec> entry in the OPTIONS section of '
780 'git-pull(1) for other possibilities.'))
781 elif command == 'push':
782 subparser.add_argument(
783 'repository', metavar='REPOSITORY', default=None, nargs='?',
785 'The "remote" repository that is the destination of the '
786 'push. This parameter can be either a URL (see the '
787 'section GIT URLS in git-push(1)) or the name of a remote '
788 '(see the section REMOTES in git-push(1)).'))
789 subparser.add_argument(
790 'refspecs', metavar='REFSPEC', default=None, nargs='*',
792 'Refspec (usually a branch name) to push. See '
793 'the <refspec> entry in the OPTIONS section of '
794 'git-push(1) for other possibilities.'))
796 args = parser.parse_args()
799 level = getattr(_logging, args.log_level.upper())
802 if not getattr(args, 'func', None):
806 if args.func == help:
807 arg_names = ['command']
809 (arg_names, varargs, varkw) = _inspect.getargs(args.func.__code__)
810 kwargs = {key: getattr(args, key) for key in arg_names if key in args}
813 except SubprocessError as e:
814 if _LOG.level == _logging.DEBUG:
815 raise # don't mask the traceback