3 # Copyright (c) 2011-2014 David Bremner <david@tethera.net>
4 # W. Trevor King <wking@tremily.us>
6 # This program is free software: you can redistribute it and/or modify
7 # it under the terms of the GNU General Public License as published by
8 # the Free Software Foundation, either version 3 of the License, or
9 # (at your option) any later version.
11 # This program is distributed in the hope that it will be useful,
12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 # GNU General Public License for more details.
16 # You should have received a copy of the GNU General Public License
17 # along with this program. If not, see https://www.gnu.org/licenses/ .
20 Manage notmuch tags with Git
22 Environment variables:
24 * NMBGIT specifies the location of the git repository used by nmbug.
25 If not specified $HOME/.nmbug is used.
26 * NMBPREFIX specifies the prefix in the notmuch database for tags of
27 interest to nmbug. If not specified 'notmuch::' is used.
30 from __future__ import print_function
31 from __future__ import unicode_literals
33 import codecs as _codecs
34 import collections as _collections
35 import functools as _functools
36 import inspect as _inspect
37 import locale as _locale
38 import logging as _logging
41 import shutil as _shutil
42 import subprocess as _subprocess
44 import tempfile as _tempfile
45 import textwrap as _textwrap
47 from urllib.parse import quote as _quote
48 from urllib.parse import unquote as _unquote
49 except ImportError: # Python 2
50 from urllib import quote as _quote
51 from urllib import unquote as _unquote
53 _LOG = _logging.getLogger('nmbug')
54 _LOG.setLevel(_logging.WARNING)
55 _LOG.addHandler(_logging.StreamHandler())
57 NMBGIT = _os.path.expanduser(
58 _os.getenv('NMBGIT', _os.path.join('~', '.nmbug')))
59 _NMBGIT = _os.path.join(NMBGIT, '.git')
60 if _os.path.isdir(_NMBGIT):
63 TAG_PREFIX = _os.getenv('NMBPREFIX', 'notmuch::')
64 _HEX_ESCAPE_REGEX = _re.compile('%[0-9A-F]{2}')
65 _TAG_DIRECTORY = 'tags/'
66 _TAG_FILE_REGEX = _re.compile(_TAG_DIRECTORY + '(?P<id>[^/]*)/(?P<tag>[^/]*)')
68 # magic hash for Git (git hash-object -t blob /dev/null)
69 _EMPTYBLOB = 'e69de29bb2d1d6434b8b29ae775ad8c2e48c5391'
73 getattr(_tempfile, 'TemporaryDirectory')
74 except AttributeError: # Python < 3.2
75 class _TemporaryDirectory(object):
77 Fallback context manager for Python < 3.2
79 See PEP 343 for details on context managers [1].
81 [1]: https://www.python.org/dev/peps/pep-0343/
83 def __init__(self, **kwargs):
84 self.name = _tempfile.mkdtemp(**kwargs)
89 def __exit__(self, type, value, traceback):
90 _shutil.rmtree(self.name)
93 _tempfile.TemporaryDirectory = _TemporaryDirectory
96 def _hex_quote(string, safe='+@=:,'):
98 quote('abc def') -> 'abc%20def'.
100 Wrap urllib.parse.quote with additional safe characters (in
101 addition to letters, digits, and '_.-') and lowercase hex digits
102 (e.g. '%3a' instead of '%3A').
104 uppercase_escapes = _quote(string, safe)
105 return _HEX_ESCAPE_REGEX.sub(
106 lambda match: match.group(0).lower(),
110 _ENCODED_TAG_PREFIX = _hex_quote(TAG_PREFIX, safe='+@=,') # quote ':'
113 def _xapian_quote(string):
115 Quote a string for Xapian's QueryParser.
117 Xapian uses double-quotes for quoting strings. You can escape
118 internal quotes by repeating them [1,2,3].
120 [1]: https://trac.xapian.org/ticket/128#comment:2
121 [2]: https://trac.xapian.org/ticket/128#comment:17
122 [3]: https://trac.xapian.org/changeset/13823/svn
124 return '"{0}"'.format(string.replace('"', '""'))
127 def _xapian_unquote(string):
129 Unquote a Xapian-quoted string.
131 if string.startswith('"') and string.endswith('"'):
132 return string[1:-1].replace('""', '"')
136 class SubprocessError(RuntimeError):
137 "A subprocess exited with a nonzero status"
138 def __init__(self, args, status, stdout=None, stderr=None):
142 msg = '{args} exited with {status}'.format(args=args, status=status)
144 msg = '{msg}: {stderr}'.format(msg=msg, stderr=stderr)
145 super(SubprocessError, self).__init__(msg)
148 class _SubprocessContextManager(object):
150 PEP 343 context manager for subprocesses.
152 'expect' holds a tuple of acceptable exit codes, otherwise we'll
153 raise a SubprocessError in __exit__.
155 def __init__(self, process, args, expect=(0,)):
156 self._process = process
158 self._expect = expect
163 def __exit__(self, type, value, traceback):
164 for name in ['stdin', 'stdout', 'stderr']:
165 stream = getattr(self._process, name)
168 setattr(self._process, name, None)
169 status = self._process.wait()
171 'collect {args} with status {status} (expected {expect})'.format(
172 args=self._args, status=status, expect=self._expect))
173 if status not in self._expect:
174 raise SubprocessError(args=self._args, status=status)
177 return self._process.wait()
180 def _spawn(args, input=None, additional_env=None, wait=False, stdin=None,
181 stdout=None, stderr=None, encoding=_locale.getpreferredencoding(),
182 expect=(0,), **kwargs):
183 """Spawn a subprocess, and optionally wait for it to finish.
185 This wrapper around subprocess.Popen has two modes, depending on
186 the truthiness of 'wait'. If 'wait' is true, we use p.communicate
187 internally to write 'input' to the subprocess's stdin and read
188 from it's stdout/stderr. If 'wait' is False, we return a
189 _SubprocessContextManager instance for fancier handling
190 (e.g. piping between processes).
192 For 'wait' calls when you want to write to the subprocess's stdin,
193 you only need to set 'input' to your content. When 'input' is not
194 None but 'stdin' is, we'll automatically set 'stdin' to PIPE
195 before calling Popen. This avoids having the subprocess
196 accidentally inherit the launching process's stdin.
198 _LOG.debug('spawn {args} (additional env. var.: {env})'.format(
199 args=args, env=additional_env))
200 if not stdin and input is not None:
201 stdin = _subprocess.PIPE
203 if not kwargs.get('env'):
204 kwargs['env'] = dict(_os.environ)
205 kwargs['env'].update(additional_env)
206 p = _subprocess.Popen(
207 args, stdin=stdin, stdout=stdout, stderr=stderr, **kwargs)
209 if hasattr(input, 'encode'):
210 input = input.encode(encoding)
211 (stdout, stderr) = p.communicate(input=input)
214 'collect {args} with status {status} (expected {expect})'.format(
215 args=args, status=status, expect=expect))
216 if stdout is not None:
217 stdout = stdout.decode(encoding)
218 if stderr is not None:
219 stderr = stderr.decode(encoding)
220 if status not in expect:
221 raise SubprocessError(
222 args=args, status=status, stdout=stdout, stderr=stderr)
223 return (status, stdout, stderr)
224 if p.stdin and not stdin:
228 p.stdin = _codecs.getwriter(encoding=encoding)(stream=p.stdin)
229 stream_reader = _codecs.getreader(encoding=encoding)
231 p.stdout = stream_reader(stream=p.stdout)
233 p.stderr = stream_reader(stream=p.stderr)
234 return _SubprocessContextManager(args=args, process=p, expect=expect)
237 def _git(args, **kwargs):
238 args = ['git', '--git-dir', NMBGIT] + list(args)
239 return _spawn(args=args, **kwargs)
242 def _get_current_branch():
243 """Get the name of the current branch.
245 Return 'None' if we're not on a branch.
248 (status, branch, stderr) = _git(
249 args=['symbolic-ref', '--short', 'HEAD'],
250 stdout=_subprocess.PIPE, stderr=_subprocess.PIPE, wait=True)
251 except SubprocessError as e:
252 if 'not a symbolic ref' in e:
255 return branch.strip()
259 "Get the default remote for the current branch."
260 local_branch = _get_current_branch()
261 (status, remote, stderr) = _git(
262 args=['config', 'branch.{0}.remote'.format(local_branch)],
263 stdout=_subprocess.PIPE, wait=True)
264 return remote.strip()
267 def get_tags(prefix=None):
268 "Get a list of tags with a given prefix."
271 (status, stdout, stderr) = _spawn(
272 args=['notmuch', 'search', '--output=tags', '*'],
273 stdout=_subprocess.PIPE, wait=True)
274 return [tag for tag in stdout.splitlines() if tag.startswith(prefix)]
277 def archive(treeish='HEAD', args=()):
279 Dump a tar archive of the current nmbug tag set.
283 Each tag $tag for message with Message-Id $id is written to
286 tags/encode($id)/encode($tag)
288 The encoding preserves alphanumerics, and the characters
289 "+-_@=.:," (not the quotes). All other octets are replaced with
290 '%' followed by a two digit hex number.
292 _git(args=['archive', treeish] + list(args), wait=True)
295 def clone(repository):
297 Create a local nmbug repository from a remote source.
299 This wraps 'git clone', adding some options to avoid creating a
300 working tree while preserving remote-tracking branches and
303 with _tempfile.TemporaryDirectory(prefix='nmbug-clone.') as workdir:
306 'git', 'clone', '--no-checkout', '--separate-git-dir', NMBGIT,
307 repository, workdir],
309 _git(args=['config', '--unset', 'core.worktree'], wait=True, expect=(0, 5))
310 _git(args=['config', 'core.bare', 'true'], wait=True)
311 _git(args=['branch', 'config', 'origin/config'], wait=True)
312 existing_tags = get_tags()
315 'Not checking out to avoid clobbering existing tags: {}'.format(
316 ', '.join(existing_tags)))
321 def _is_committed(status):
322 return len(status['added']) + len(status['deleted']) == 0
325 def commit(treeish='HEAD', message=None):
327 Commit prefix-matching tags from the notmuch database to Git.
329 status = get_status()
331 if _is_committed(status=status):
332 _LOG.warning('Nothing to commit')
335 _git(args=['read-tree', '--empty'], wait=True)
336 _git(args=['read-tree', treeish], wait=True)
338 _update_index(status=status)
341 stdout=_subprocess.PIPE,
343 (_, parent, _) = _git(
344 args=['rev-parse', treeish],
345 stdout=_subprocess.PIPE,
347 (_, commit, _) = _git(
348 args=['commit-tree', tree.strip(), '-p', parent.strip()],
350 stdout=_subprocess.PIPE,
353 args=['update-ref', treeish, commit.strip()],
354 stdout=_subprocess.PIPE,
356 except Exception as e:
357 _git(args=['read-tree', '--empty'], wait=True)
358 _git(args=['read-tree', treeish], wait=True)
361 def _update_index(status):
363 args=['update-index', '--index-info'],
364 stdin=_subprocess.PIPE) as p:
365 for id, tags in status['deleted'].items():
366 for line in _index_tags_for_message(id=id, status='D', tags=tags):
368 for id, tags in status['added'].items():
369 for line in _index_tags_for_message(id=id, status='A', tags=tags):
373 def fetch(remote=None):
375 Fetch changes from the remote repository.
377 See 'merge' to bring those changes into notmuch.
382 _git(args=args, wait=True)
387 Update the notmuch database from Git.
389 This is mainly useful to discard your changes in notmuch relative
392 status = get_status()
394 args=['notmuch', 'tag', '--batch'], stdin=_subprocess.PIPE) as p:
395 for id, tags in status['added'].items():
396 p.stdin.write(_batch_line(action='-', id=id, tags=tags))
397 for id, tags in status['deleted'].items():
398 p.stdin.write(_batch_line(action='+', id=id, tags=tags))
401 def _batch_line(action, id, tags):
403 'notmuch tag --batch' line for adding/removing tags.
405 Set 'action' to '-' to remove a tag or '+' to add the tags to a
408 tag_string = ' '.join(
409 '{action}{prefix}{tag}'.format(
410 action=action, prefix=_ENCODED_TAG_PREFIX, tag=_hex_quote(tag))
412 line = '{tags} -- id:{id}\n'.format(
413 tags=tag_string, id=_xapian_quote(string=id))
417 def _insist_committed():
418 "Die if the the notmuch tags don't match the current HEAD."
419 status = get_status()
420 if not _is_committed(status=status):
421 _LOG.error('\n'.join([
422 'Uncommitted changes to {prefix}* tags in notmuch',
424 "For a summary of changes, run 'nmbug status'",
425 "To save your changes, run 'nmbug commit' before merging/pull",
426 "To discard your changes, run 'nmbug checkout'",
427 ]).format(prefix=TAG_PREFIX))
431 def pull(repository=None, refspecs=None):
433 Pull (merge) remote repository changes to notmuch.
435 'pull' is equivalent to 'fetch' followed by 'merge'. We use the
436 Git-configured repository for your current branch
437 (branch.<name>.repository, likely 'origin', and
438 branch.<name>.merge, likely 'master').
441 if refspecs and not repository:
442 repository = _get_remote()
445 args.append(repository)
447 args.extend(refspecs)
448 with _tempfile.TemporaryDirectory(prefix='nmbug-pull.') as workdir:
454 additional_env={'GIT_WORK_TREE': workdir},
459 def merge(reference='@{upstream}'):
461 Merge changes from 'reference' into HEAD and load the result into notmuch.
463 The default reference is '@{upstream}'.
466 with _tempfile.TemporaryDirectory(prefix='nmbug-merge.') as workdir:
469 ['merge', reference]]:
472 additional_env={'GIT_WORK_TREE': workdir},
479 A simple wrapper for 'git log'.
481 After running 'nmbug fetch', you can inspect the changes with
482 'nmbug log HEAD..@{upstream}'.
484 # we don't want output trapping here, because we want the pager.
485 args = ['log', '--name-status', '--no-renames'] + list(args)
486 with _git(args=args, expect=(0, 1, -13)) as p:
490 def push(repository=None, refspecs=None):
491 "Push the local nmbug Git state to a remote repository."
492 if refspecs and not repository:
493 repository = _get_remote()
496 args.append(repository)
498 args.extend(refspecs)
499 _git(args=args, wait=True)
504 Show pending updates in notmuch or git repo.
506 Prints lines of the form
510 where n is a single character representing notmuch database status
514 Tag is present in notmuch database, but not committed to nmbug
515 (equivalently, tag has been deleted in nmbug repo, e.g. by a
516 pull, but not restored to notmuch database).
520 Tag is present in nmbug repo, but not restored to notmuch
521 database (equivalently, tag has been deleted in notmuch).
525 Message is unknown (missing from local notmuch database).
527 The second character (if present) represents a difference between
528 local and upstream branches. Typically 'nmbug fetch' needs to be
533 Tag is present in upstream, but not in the local Git branch.
537 Tag is present in local Git branch, but not upstream.
539 status = get_status()
540 # 'output' is a nested defaultdict for message status:
541 # * The outer dict is keyed by message id.
542 # * The inner dict is keyed by tag name.
543 # * The inner dict values are status strings (' a', 'Dd', ...).
544 output = _collections.defaultdict(
545 lambda : _collections.defaultdict(lambda : ' '))
546 for id, tags in status['added'].items():
548 output[id][tag] = 'A'
549 for id, tags in status['deleted'].items():
551 output[id][tag] = 'D'
552 for id, tags in status['missing'].items():
554 output[id][tag] = 'U'
556 for id, tag in _diff_refs(filter='A'):
557 output[id][tag] += 'a'
558 for id, tag in _diff_refs(filter='D'):
559 output[id][tag] += 'd'
560 for id, tag_status in sorted(output.items()):
561 for tag, status in sorted(tag_status.items()):
562 print('{status}\t{id}\t{tag}'.format(
563 status=status, id=id, tag=tag))
566 def _is_unmerged(ref='@{upstream}'):
568 (status, fetch_head, stderr) = _git(
569 args=['rev-parse', ref],
570 stdout=_subprocess.PIPE, stderr=_subprocess.PIPE, wait=True)
571 except SubprocessError as e:
572 if 'No upstream configured' in e.stderr:
575 (status, base, stderr) = _git(
576 args=['merge-base', 'HEAD', ref],
577 stdout=_subprocess.PIPE, wait=True)
578 return base != fetch_head
586 index = _index_tags()
587 maybe_deleted = _diff_index(index=index, filter='D')
588 for id, tags in maybe_deleted.items():
589 (_, stdout, stderr) = _spawn(
590 args=['notmuch', 'search', '--output=files', 'id:{0}'.format(id)],
591 stdout=_subprocess.PIPE,
594 status['deleted'][id] = tags
596 status['missing'][id] = tags
597 status['added'] = _diff_index(index=index, filter='A')
603 "Write notmuch tags to the nmbug.index."
604 path = _os.path.join(NMBGIT, 'nmbug.index')
605 query = ' '.join('tag:"{tag}"'.format(tag=tag) for tag in get_tags())
606 prefix = '+{0}'.format(_ENCODED_TAG_PREFIX)
608 args=['read-tree', '--empty'],
609 additional_env={'GIT_INDEX_FILE': path}, wait=True)
611 args=['notmuch', 'dump', '--format=batch-tag', '--', query],
612 stdout=_subprocess.PIPE) as notmuch:
614 args=['update-index', '--index-info'],
615 stdin=_subprocess.PIPE,
616 additional_env={'GIT_INDEX_FILE': path}) as git:
617 for line in notmuch.stdout:
618 if line.strip().startswith('#'):
620 (tags_string, id) = [_.strip() for _ in line.split(' -- id:')]
622 _unquote(tag[len(prefix):])
623 for tag in tags_string.split()
624 if tag.startswith(prefix)]
625 id = _xapian_unquote(string=id)
626 for line in _index_tags_for_message(
627 id=id, status='A', tags=tags):
628 git.stdin.write(line)
632 def _index_tags_for_message(id, status, tags):
634 Update the Git index to either create or delete an empty file.
636 Neither 'id' nor the tags in 'tags' should be encoded/escaped.
643 hash = '0000000000000000000000000000000000000000'
646 path = 'tags/{id}/{tag}'.format(
647 id=_hex_quote(string=id), tag=_hex_quote(string=tag))
648 yield '{mode} {hash}\t{path}\n'.format(mode=mode, hash=hash, path=path)
651 def _diff_index(index, filter):
653 Get an {id: {tag, ...}} dict for a given filter.
655 For example, use 'A' to find added tags, and 'D' to find deleted tags.
657 s = _collections.defaultdict(set)
660 'diff-index', '--cached', '--diff-filter', filter,
661 '--name-only', 'HEAD'],
662 additional_env={'GIT_INDEX_FILE': index},
663 stdout=_subprocess.PIPE) as p:
664 # Once we drop Python < 3.3, we can use 'yield from' here
665 for id, tag in _unpack_diff_lines(stream=p.stdout):
670 def _diff_refs(filter, a='HEAD', b='@{upstream}'):
672 args=['diff', '--diff-filter', filter, '--name-only', a, b],
673 stdout=_subprocess.PIPE) as p:
674 # Once we drop Python < 3.3, we can use 'yield from' here
675 for id, tag in _unpack_diff_lines(stream=p.stdout):
679 def _unpack_diff_lines(stream):
680 "Iterate through (id, tag) tuples in a diff stream."
682 match = _TAG_FILE_REGEX.match(line.strip())
684 message = 'non-tag line in diff: {!r}'.format(line.strip())
685 if line.startswith(_TAG_DIRECTORY):
686 raise ValueError(message)
689 id = _unquote(match.group('id'))
690 tag = _unquote(match.group('tag'))
694 def _help(parser, command=None):
696 Show help for an nmbug command.
698 Because some folks prefer:
704 $ nmbug COMMAND --help
707 parser.parse_args([command, '--help'])
709 parser.parse_args(['--help'])
712 if __name__ == '__main__':
715 parser = argparse.ArgumentParser(
716 description=__doc__.strip(),
717 formatter_class=argparse.RawDescriptionHelpFormatter)
720 choices=['critical', 'error', 'warning', 'info', 'debug'],
721 help='Log verbosity. Defaults to {!r}.'.format(
722 _logging.getLevelName(_LOG.level).lower()))
724 help = _functools.partial(_help, parser=parser)
725 help.__doc__ = _help.__doc__
726 subparsers = parser.add_subparsers(
729 'For help on a particular command, run: '
730 "'%(prog)s ... <command> --help'."))
744 func = locals()[command]
745 doc = _textwrap.dedent(func.__doc__).strip().replace('%', '%%')
746 subparser = subparsers.add_parser(
748 help=doc.splitlines()[0],
750 formatter_class=argparse.RawDescriptionHelpFormatter)
751 subparser.set_defaults(func=func)
752 if command == 'archive':
753 subparser.add_argument(
754 'treeish', metavar='TREE-ISH', nargs='?', default='HEAD',
756 'The tree or commit to produce an archive for. Defaults '
758 subparser.add_argument(
759 'args', metavar='ARG', nargs='*',
761 "Argument passed through to 'git archive'. Set anything "
762 'before <tree-ish>, see git-archive(1) for details.'))
763 elif command == 'clone':
764 subparser.add_argument(
767 'The (possibly remote) repository to clone from. See the '
768 'URLS section of git-clone(1) for more information on '
769 'specifying repositories.'))
770 elif command == 'commit':
771 subparser.add_argument(
772 'message', metavar='MESSAGE', default='', nargs='?',
773 help='Text for the commit message.')
774 elif command == 'fetch':
775 subparser.add_argument(
776 'remote', metavar='REMOTE', nargs='?',
778 'Override the default configured in branch.<name>.remote '
779 'to fetch from a particular remote repository (e.g. '
781 elif command == 'help':
782 subparser.add_argument(
783 'command', metavar='COMMAND', nargs='?',
784 help='The command to show help for.')
785 elif command == 'log':
786 subparser.add_argument(
787 'args', metavar='ARG', nargs='*',
788 help="Additional argument passed through to 'git log'.")
789 elif command == 'merge':
790 subparser.add_argument(
791 'reference', metavar='REFERENCE', default='@{upstream}',
794 'Reference, usually other branch heads, to merge into '
795 "our branch. Defaults to '@{upstream}'."))
796 elif command == 'pull':
797 subparser.add_argument(
798 'repository', metavar='REPOSITORY', default=None, nargs='?',
800 'The "remote" repository that is the source of the pull. '
801 'This parameter can be either a URL (see the section GIT '
802 'URLS in git-pull(1)) or the name of a remote (see the '
803 'section REMOTES in git-pull(1)).'))
804 subparser.add_argument(
805 'refspecs', metavar='REFSPEC', default=None, nargs='*',
807 'Refspec (usually a branch name) to fetch and merge. See '
808 'the <refspec> entry in the OPTIONS section of '
809 'git-pull(1) for other possibilities.'))
810 elif command == 'push':
811 subparser.add_argument(
812 'repository', metavar='REPOSITORY', default=None, nargs='?',
814 'The "remote" repository that is the destination of the '
815 'push. This parameter can be either a URL (see the '
816 'section GIT URLS in git-push(1)) or the name of a remote '
817 '(see the section REMOTES in git-push(1)).'))
818 subparser.add_argument(
819 'refspecs', metavar='REFSPEC', default=None, nargs='*',
821 'Refspec (usually a branch name) to push. See '
822 'the <refspec> entry in the OPTIONS section of '
823 'git-push(1) for other possibilities.'))
825 args = parser.parse_args()
828 level = getattr(_logging, args.log_level.upper())
831 if not getattr(args, 'func', None):
835 if args.func == help:
836 arg_names = ['command']
838 (arg_names, varargs, varkw) = _inspect.getargs(args.func.__code__)
839 kwargs = {key: getattr(args, key) for key in arg_names if key in args}
842 except SubprocessError as e:
843 if _LOG.level == _logging.DEBUG:
844 raise # don't mask the traceback