3 # Copyright (c) 2011-2014 David Bremner <david@tethera.net>
4 # W. Trevor King <wking@tremily.us>
6 # This program is free software: you can redistribute it and/or modify
7 # it under the terms of the GNU General Public License as published by
8 # the Free Software Foundation, either version 3 of the License, or
9 # (at your option) any later version.
11 # This program is distributed in the hope that it will be useful,
12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 # GNU General Public License for more details.
16 # You should have received a copy of the GNU General Public License
17 # along with this program. If not, see https://www.gnu.org/licenses/ .
20 Manage notmuch tags with Git
22 Environment variables:
24 * NMBGIT specifies the location of the git repository used by nmbug.
25 If not specified $HOME/.nmbug is used.
26 * NMBPREFIX specifies the prefix in the notmuch database for tags of
27 interest to nmbug. If not specified 'notmuch::' is used.
30 from __future__ import print_function
31 from __future__ import unicode_literals
33 import codecs as _codecs
34 import collections as _collections
35 import functools as _functools
36 import inspect as _inspect
37 import locale as _locale
38 import logging as _logging
41 import shutil as _shutil
42 import subprocess as _subprocess
44 import tempfile as _tempfile
45 import textwrap as _textwrap
47 from urllib.parse import quote as _quote
48 from urllib.parse import unquote as _unquote
49 except ImportError: # Python 2
50 from urllib import quote as _quote
51 from urllib import unquote as _unquote
56 _LOG = _logging.getLogger('nmbug')
57 _LOG.setLevel(_logging.ERROR)
58 _LOG.addHandler(_logging.StreamHandler())
60 NMBGIT = _os.path.expanduser(
61 _os.getenv('NMBGIT', _os.path.join('~', '.nmbug')))
62 _NMBGIT = _os.path.join(NMBGIT, '.git')
63 if _os.path.isdir(_NMBGIT):
66 TAG_PREFIX = _os.getenv('NMBPREFIX', 'notmuch::')
67 _HEX_ESCAPE_REGEX = _re.compile('%[0-9A-F]{2}')
68 _TAG_FILE_REGEX = _re.compile('tags/(?P<id>[^/]*)/(?P<tag>[^/]*)')
70 # magic hash for Git (git hash-object -t blob /dev/null)
71 _EMPTYBLOB = 'e69de29bb2d1d6434b8b29ae775ad8c2e48c5391'
75 getattr(_tempfile, 'TemporaryDirectory')
76 except AttributeError: # Python < 3.2
77 class _TemporaryDirectory(object):
79 Fallback context manager for Python < 3.2
81 See PEP 343 for details on context managers [1].
83 [1]: https://www.python.org/dev/peps/pep-0343/
85 def __init__(self, **kwargs):
86 self.name = _tempfile.mkdtemp(**kwargs)
91 def __exit__(self, type, value, traceback):
92 _shutil.rmtree(self.name)
95 _tempfile.TemporaryDirectory = _TemporaryDirectory
98 def _hex_quote(string, safe='+@=:,'):
100 quote('abc def') -> 'abc%20def'.
102 Wrap urllib.parse.quote with additional safe characters (in
103 addition to letters, digits, and '_.-') and lowercase hex digits
104 (e.g. '%3a' instead of '%3A').
106 uppercase_escapes = _quote(string, safe)
107 return _HEX_ESCAPE_REGEX.sub(
108 lambda match: match.group(0).lower(),
112 _ENCODED_TAG_PREFIX = _hex_quote(TAG_PREFIX, safe='+@=,') # quote ':'
115 def _xapian_quote(string):
117 Quote a string for Xapian's QueryParser.
119 Xapian uses double-quotes for quoting strings. You can escape
120 internal quotes by repeating them [1,2,3].
122 [1]: https://trac.xapian.org/ticket/128#comment:2
123 [2]: https://trac.xapian.org/ticket/128#comment:17
124 [3]: https://trac.xapian.org/changeset/13823/svn
126 return '"{0}"'.format(string.replace('"', '""'))
129 def _xapian_unquote(string):
131 Unquote a Xapian-quoted string.
133 if string.startswith('"') and string.endswith('"'):
134 return string[1:-1].replace('""', '"')
138 class SubprocessError(RuntimeError):
139 "A subprocess exited with a nonzero status"
140 def __init__(self, args, status, stdout=None, stderr=None):
144 msg = '{args} exited with {status}'.format(args=args, status=status)
146 msg = '{msg}: {stderr}'.format(msg=msg, stderr=stderr)
147 super(SubprocessError, self).__init__(msg)
150 class _SubprocessContextManager(object):
152 PEP 343 context manager for subprocesses.
154 'expect' holds a tuple of acceptable exit codes, otherwise we'll
155 raise a SubprocessError in __exit__.
157 def __init__(self, process, args, expect=(0,)):
158 self._process = process
160 self._expect = expect
165 def __exit__(self, type, value, traceback):
166 for name in ['stdin', 'stdout', 'stderr']:
167 stream = getattr(self._process, name)
170 setattr(self._process, name, None)
171 status = self._process.wait()
172 _LOG.debug('collect {args} with status {status}'.format(
173 args=self._args, status=status))
174 if status not in self._expect:
175 raise SubprocessError(args=self._args, status=status)
178 return self._process.wait()
181 def _spawn(args, input=None, additional_env=None, wait=False, stdin=None,
182 stdout=None, stderr=None, encoding=_locale.getpreferredencoding(),
183 expect=(0,), **kwargs):
184 """Spawn a subprocess, and optionally wait for it to finish.
186 This wrapper around subprocess.Popen has two modes, depending on
187 the truthiness of 'wait'. If 'wait' is true, we use p.communicate
188 internally to write 'input' to the subprocess's stdin and read
189 from it's stdout/stderr. If 'wait' is False, we return a
190 _SubprocessContextManager instance for fancier handling
191 (e.g. piping between processes).
193 For 'wait' calls when you want to write to the subprocess's stdin,
194 you only need to set 'input' to your content. When 'input' is not
195 None but 'stdin' is, we'll automatically set 'stdin' to PIPE
196 before calling Popen. This avoids having the subprocess
197 accidentally inherit the launching process's stdin.
199 _LOG.debug('spawn {args} (additional env. var.: {env})'.format(
200 args=args, env=additional_env))
201 if not stdin and input is not None:
202 stdin = _subprocess.PIPE
204 if not kwargs.get('env'):
205 kwargs['env'] = dict(_os.environ)
206 kwargs['env'].update(additional_env)
207 p = _subprocess.Popen(
208 args, stdin=stdin, stdout=stdout, stderr=stderr, **kwargs)
210 if hasattr(input, 'encode'):
211 input = input.encode(encoding)
212 (stdout, stderr) = p.communicate(input=input)
214 _LOG.debug('collect {args} with status {status}'.format(
215 args=args, status=status))
216 if stdout is not None:
217 stdout = stdout.decode(encoding)
218 if stderr is not None:
219 stderr = stderr.decode(encoding)
221 raise SubprocessError(
222 args=args, status=status, stdout=stdout, stderr=stderr)
223 return (status, stdout, stderr)
224 if p.stdin and not stdin:
228 p.stdin = _codecs.getwriter(encoding=encoding)(stream=p.stdin)
229 stream_reader = _codecs.getreader(encoding=encoding)
231 p.stdout = stream_reader(stream=p.stdout)
233 p.stderr = stream_reader(stream=p.stderr)
234 return _SubprocessContextManager(args=args, process=p, expect=expect)
237 def _git(args, **kwargs):
238 args = ['git', '--git-dir', NMBGIT] + list(args)
239 return _spawn(args=args, **kwargs)
242 def _get_current_branch():
243 """Get the name of the current branch.
245 Return 'None' if we're not on a branch.
248 (status, branch, stderr) = _git(
249 args=['symbolic-ref', '--short', 'HEAD'],
250 stdout=_subprocess.PIPE, stderr=_subprocess.PIPE, wait=True)
251 except SubprocessError as e:
252 if 'not a symbolic ref' in e:
255 return branch.strip()
259 "Get the default remote for the current branch."
260 local_branch = _get_current_branch()
261 (status, remote, stderr) = _git(
262 args=['config', 'branch.{0}.remote'.format(local_branch)],
263 stdout=_subprocess.PIPE, wait=True)
264 return remote.strip()
267 def get_tags(prefix=None):
268 "Get a list of tags with a given prefix."
271 (status, stdout, stderr) = _spawn(
272 args=['notmuch', 'search', '--output=tags', '*'],
273 stdout=_subprocess.PIPE, wait=True)
274 return [tag for tag in stdout.splitlines() if tag.startswith(prefix)]
277 def archive(treeish='HEAD', args=()):
279 Dump a tar archive of the current nmbug tag set.
283 Each tag $tag for message with Message-Id $id is written to
286 tags/encode($id)/encode($tag)
288 The encoding preserves alphanumerics, and the characters
289 "+-_@=.:," (not the quotes). All other octets are replaced with
290 '%' followed by a two digit hex number.
292 _git(args=['archive', treeish] + list(args), wait=True)
295 def clone(repository):
297 Create a local nmbug repository from a remote source.
299 This wraps 'git clone', adding some options to avoid creating a
300 working tree while preserving remote-tracking branches and
303 with _tempfile.TemporaryDirectory(prefix='nmbug-clone.') as workdir:
306 'git', 'clone', '--no-checkout', '--separate-git-dir', NMBGIT,
307 repository, workdir],
309 _git(args=['config', '--unset', 'core.worktree'], wait=True)
310 _git(args=['config', 'core.bare', 'true'], wait=True)
311 _git(args=['branch', 'config', 'origin/config'], wait=True)
314 def _is_committed(status):
315 return len(status['added']) + len(status['deleted']) == 0
318 def commit(treeish='HEAD', message=None):
320 Commit prefix-matching tags from the notmuch database to Git.
322 status = get_status()
324 if _is_committed(status=status):
325 _LOG.warning('Nothing to commit')
328 _git(args=['read-tree', '--empty'], wait=True)
329 _git(args=['read-tree', treeish], wait=True)
331 _update_index(status=status)
334 stdout=_subprocess.PIPE,
336 (_, parent, _) = _git(
337 args=['rev-parse', treeish],
338 stdout=_subprocess.PIPE,
340 (_, commit, _) = _git(
341 args=['commit-tree', tree.strip(), '-p', parent.strip()],
343 stdout=_subprocess.PIPE,
346 args=['update-ref', treeish, commit.strip()],
347 stdout=_subprocess.PIPE,
349 except Exception as e:
350 _git(args=['read-tree', '--empty'], wait=True)
351 _git(args=['read-tree', treeish], wait=True)
354 def _update_index(status):
356 args=['update-index', '--index-info'],
357 stdin=_subprocess.PIPE) as p:
358 for id, tags in status['deleted'].items():
359 for line in _index_tags_for_message(id=id, status='D', tags=tags):
361 for id, tags in status['added'].items():
362 for line in _index_tags_for_message(id=id, status='A', tags=tags):
366 def fetch(remote=None):
368 Fetch changes from the remote repository.
370 See 'merge' to bring those changes into notmuch.
375 _git(args=args, wait=True)
380 Update the notmuch database from Git.
382 This is mainly useful to discard your changes in notmuch relative
385 status = get_status()
387 args=['notmuch', 'tag', '--batch'], stdin=_subprocess.PIPE) as p:
388 for id, tags in status['added'].items():
389 p.stdin.write(_batch_line(action='-', id=id, tags=tags))
390 for id, tags in status['deleted'].items():
391 p.stdin.write(_batch_line(action='+', id=id, tags=tags))
394 def _batch_line(action, id, tags):
396 'notmuch tag --batch' line for adding/removing tags.
398 Set 'action' to '-' to remove a tag or '+' to add the tags to a
401 tag_string = ' '.join(
402 '{action}{prefix}{tag}'.format(
403 action=action, prefix=_ENCODED_TAG_PREFIX, tag=_hex_quote(tag))
405 line = '{tags} -- id:{id}\n'.format(
406 tags=tag_string, id=_xapian_quote(string=id))
410 def _insist_committed():
411 "Die if the the notmuch tags don't match the current HEAD."
412 status = get_status()
413 if not _is_committed(status=status):
414 _LOG.error('\n'.join([
415 'Uncommitted changes to {prefix}* tags in notmuch',
417 "For a summary of changes, run 'nmbug status'",
418 "To save your changes, run 'nmbug commit' before merging/pull",
419 "To discard your changes, run 'nmbug checkout'",
420 ]).format(prefix=TAG_PREFIX))
424 def pull(repository=None, refspecs=None):
426 Pull (merge) remote repository changes to notmuch.
428 'pull' is equivalent to 'fetch' followed by 'merge'. We use the
429 Git-configured repository for your current branch
430 (branch.<name>.repository, likely 'origin', and
431 branch.<name>.merge, likely 'master').
434 if refspecs and not repository:
435 repository = _get_remote()
438 args.append(repository)
440 args.extend(refspecs)
441 with _tempfile.TemporaryDirectory(prefix='nmbug-pull.') as workdir:
447 additional_env={'GIT_WORK_TREE': workdir},
452 def merge(reference='@{upstream}'):
454 Merge changes from 'reference' into HEAD and load the result into notmuch.
456 The default reference is '@{upstream}'.
459 with _tempfile.TemporaryDirectory(prefix='nmbug-merge.') as workdir:
462 ['merge', reference]]:
465 additional_env={'GIT_WORK_TREE': workdir},
472 A simple wrapper for 'git log'.
474 After running 'nmbug fetch', you can inspect the changes with
475 'nmbug log HEAD..@{upstream}'.
477 # we don't want output trapping here, because we want the pager.
478 args = ['log', '--name-status', '--no-renames'] + list(args)
479 with _git(args=args, expect=(0, 1, -13)) as p:
483 def push(repository=None, refspecs=None):
484 "Push the local nmbug Git state to a remote repository."
485 if refspecs and not repository:
486 repository = _get_remote()
489 args.append(repository)
491 args.extend(refspecs)
492 _git(args=args, wait=True)
497 Show pending updates in notmuch or git repo.
499 Prints lines of the form
503 where n is a single character representing notmuch database status
507 Tag is present in notmuch database, but not committed to nmbug
508 (equivalently, tag has been deleted in nmbug repo, e.g. by a
509 pull, but not restored to notmuch database).
513 Tag is present in nmbug repo, but not restored to notmuch
514 database (equivalently, tag has been deleted in notmuch).
518 Message is unknown (missing from local notmuch database).
520 The second character (if present) represents a difference between
521 local and upstream branches. Typically 'nmbug fetch' needs to be
526 Tag is present in upstream, but not in the local Git branch.
530 Tag is present in local Git branch, but not upstream.
532 status = get_status()
533 # 'output' is a nested defaultdict for message status:
534 # * The outer dict is keyed by message id.
535 # * The inner dict is keyed by tag name.
536 # * The inner dict values are status strings (' a', 'Dd', ...).
537 output = _collections.defaultdict(
538 lambda : _collections.defaultdict(lambda : ' '))
539 for id, tags in status['added'].items():
541 output[id][tag] = 'A'
542 for id, tags in status['deleted'].items():
544 output[id][tag] = 'D'
545 for id, tags in status['missing'].items():
547 output[id][tag] = 'U'
549 for id, tag in _diff_refs(filter='A'):
550 output[id][tag] += 'a'
551 for id, tag in _diff_refs(filter='D'):
552 output[id][tag] += 'd'
553 for id, tag_status in sorted(output.items()):
554 for tag, status in sorted(tag_status.items()):
555 print('{status}\t{id}\t{tag}'.format(
556 status=status, id=id, tag=tag))
559 def _is_unmerged(ref='@{upstream}'):
561 (status, fetch_head, stderr) = _git(
562 args=['rev-parse', ref],
563 stdout=_subprocess.PIPE, stderr=_subprocess.PIPE, wait=True)
564 except SubprocessError as e:
565 if 'No upstream configured' in e.stderr:
568 (status, base, stderr) = _git(
569 args=['merge-base', 'HEAD', ref],
570 stdout=_subprocess.PIPE, wait=True)
571 return base != fetch_head
579 index = _index_tags()
580 maybe_deleted = _diff_index(index=index, filter='D')
581 for id, tags in maybe_deleted.items():
582 (_, stdout, stderr) = _spawn(
583 args=['notmuch', 'search', '--output=files', 'id:{0}'.format(id)],
584 stdout=_subprocess.PIPE,
587 status['deleted'][id] = tags
589 status['missing'][id] = tags
590 status['added'] = _diff_index(index=index, filter='A')
596 "Write notmuch tags to the nmbug.index."
597 path = _os.path.join(NMBGIT, 'nmbug.index')
598 query = ' '.join('tag:"{tag}"'.format(tag=tag) for tag in get_tags())
599 prefix = '+{0}'.format(_ENCODED_TAG_PREFIX)
601 args=['read-tree', '--empty'],
602 additional_env={'GIT_INDEX_FILE': path}, wait=True)
604 args=['notmuch', 'dump', '--format=batch-tag', '--', query],
605 stdout=_subprocess.PIPE) as notmuch:
607 args=['update-index', '--index-info'],
608 stdin=_subprocess.PIPE,
609 additional_env={'GIT_INDEX_FILE': path}) as git:
610 for line in notmuch.stdout:
611 if line.strip().startswith('#'):
613 (tags_string, id) = [_.strip() for _ in line.split(' -- id:')]
615 _unquote(tag[len(prefix):])
616 for tag in tags_string.split()
617 if tag.startswith(prefix)]
618 id = _xapian_unquote(string=id)
619 for line in _index_tags_for_message(
620 id=id, status='A', tags=tags):
621 git.stdin.write(line)
625 def _index_tags_for_message(id, status, tags):
627 Update the Git index to either create or delete an empty file.
629 Neither 'id' nor the tags in 'tags' should be encoded/escaped.
636 hash = '0000000000000000000000000000000000000000'
639 path = 'tags/{id}/{tag}'.format(
640 id=_hex_quote(string=id), tag=_hex_quote(string=tag))
641 yield '{mode} {hash}\t{path}\n'.format(mode=mode, hash=hash, path=path)
644 def _diff_index(index, filter):
646 Get an {id: {tag, ...}} dict for a given filter.
648 For example, use 'A' to find added tags, and 'D' to find deleted tags.
650 s = _collections.defaultdict(set)
653 'diff-index', '--cached', '--diff-filter', filter,
654 '--name-only', 'HEAD'],
655 additional_env={'GIT_INDEX_FILE': index},
656 stdout=_subprocess.PIPE) as p:
657 # Once we drop Python < 3.3, we can use 'yield from' here
658 for id, tag in _unpack_diff_lines(stream=p.stdout):
663 def _diff_refs(filter, a='HEAD', b='@{upstream}'):
665 args=['diff', '--diff-filter', filter, '--name-only', a, b],
666 stdout=_subprocess.PIPE) as p:
667 # Once we drop Python < 3.3, we can use 'yield from' here
668 for id, tag in _unpack_diff_lines(stream=p.stdout):
672 def _unpack_diff_lines(stream):
673 "Iterate through (id, tag) tuples in a diff stream."
675 match = _TAG_FILE_REGEX.match(line.strip())
678 'Invalid line in diff: {!r}'.format(line.strip()))
679 id = _unquote(match.group('id'))
680 tag = _unquote(match.group('tag'))
684 def _help(parser, command=None):
686 Show help for an nmbug command.
688 Because some folks prefer:
694 $ nmbug COMMAND --help
697 parser.parse_args([command, '--help'])
699 parser.parse_args(['--help'])
702 if __name__ == '__main__':
705 parser = argparse.ArgumentParser(
706 description=__doc__.strip(),
707 formatter_class=argparse.RawDescriptionHelpFormatter)
709 '-v', '--version', action='version',
710 version='%(prog)s {}'.format(__version__))
713 choices=['critical', 'error', 'warning', 'info', 'debug'],
714 help='Log verbosity. Defaults to {!r}.'.format(
715 _logging.getLevelName(_LOG.level).lower()))
717 help = _functools.partial(_help, parser=parser)
718 help.__doc__ = _help.__doc__
719 subparsers = parser.add_subparsers(
722 'For help on a particular command, run: '
723 "'%(prog)s ... <command> --help'."))
737 func = locals()[command]
738 doc = _textwrap.dedent(func.__doc__).strip().replace('%', '%%')
739 subparser = subparsers.add_parser(
741 help=doc.splitlines()[0],
743 formatter_class=argparse.RawDescriptionHelpFormatter)
744 subparser.set_defaults(func=func)
745 if command == 'archive':
746 subparser.add_argument(
747 'treeish', metavar='TREE-ISH', nargs='?', default='HEAD',
749 'The tree or commit to produce an archive for. Defaults '
751 subparser.add_argument(
752 'args', metavar='ARG', nargs='*',
754 "Argument passed through to 'git archive'. Set anything "
755 'before <tree-ish>, see git-archive(1) for details.'))
756 elif command == 'clone':
757 subparser.add_argument(
760 'The (possibly remote) repository to clone from. See the '
761 'URLS section of git-clone(1) for more information on '
762 'specifying repositories.'))
763 elif command == 'commit':
764 subparser.add_argument(
765 'message', metavar='MESSAGE', default='', nargs='?',
766 help='Text for the commit message.')
767 elif command == 'fetch':
768 subparser.add_argument(
769 'remote', metavar='REMOTE', nargs='?',
771 'Override the default configured in branch.<name>.remote '
772 'to fetch from a particular remote repository (e.g. '
774 elif command == 'help':
775 subparser.add_argument(
776 'command', metavar='COMMAND', nargs='?',
777 help='The command to show help for.')
778 elif command == 'log':
779 subparser.add_argument(
780 'args', metavar='ARG', nargs='*',
781 help="Additional argument passed through to 'git log'.")
782 elif command == 'merge':
783 subparser.add_argument(
784 'reference', metavar='REFERENCE', default='@{upstream}',
787 'Reference, usually other branch heads, to merge into '
788 "our branch. Defaults to '@{upstream}'."))
789 elif command == 'pull':
790 subparser.add_argument(
791 'repository', metavar='REPOSITORY', default=None, nargs='?',
793 'The "remote" repository that is the source of the pull. '
794 'This parameter can be either a URL (see the section GIT '
795 'URLS in git-pull(1)) or the name of a remote (see the '
796 'section REMOTES in git-pull(1)).'))
797 subparser.add_argument(
798 'refspecs', metavar='REFSPEC', default=None, nargs='*',
800 'Refspec (usually a branch name) to fetch and merge. See '
801 'the <refspec> entry in the OPTIONS section of '
802 'git-pull(1) for other possibilities.'))
803 elif command == 'push':
804 subparser.add_argument(
805 'repository', metavar='REPOSITORY', default=None, nargs='?',
807 'The "remote" repository that is the destination of the '
808 'push. This parameter can be either a URL (see the '
809 'section GIT URLS in git-push(1)) or the name of a remote '
810 '(see the section REMOTES in git-push(1)).'))
811 subparser.add_argument(
812 'refspecs', metavar='REFSPEC', default=None, nargs='*',
814 'Refspec (usually a branch name) to push. See '
815 'the <refspec> entry in the OPTIONS section of '
816 'git-push(1) for other possibilities.'))
818 args = parser.parse_args()
821 level = getattr(_logging, args.log_level.upper())
824 if not getattr(args, 'func', None):
828 if args.func == help:
829 arg_names = ['command']
831 (arg_names, varargs, varkw) = _inspect.getargs(args.func.__code__)
832 kwargs = {key: getattr(args, key) for key in arg_names if key in args}
835 except SubprocessError as e:
836 if _LOG.level == _logging.DEBUG:
837 raise # don't mask the traceback