3 # Copyright (c) 2011-2014 David Bremner <david@tethera.net>
4 # W. Trevor King <wking@tremily.us>
6 # This program is free software: you can redistribute it and/or modify
7 # it under the terms of the GNU General Public License as published by
8 # the Free Software Foundation, either version 3 of the License, or
9 # (at your option) any later version.
11 # This program is distributed in the hope that it will be useful,
12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 # GNU General Public License for more details.
16 # You should have received a copy of the GNU General Public License
17 # along with this program. If not, see https://www.gnu.org/licenses/ .
20 Manage notmuch tags with Git
22 Environment variables:
24 * NMBGIT specifies the location of the git repository used by nmbug.
25 If not specified $HOME/.nmbug is used.
26 * NMBPREFIX specifies the prefix in the notmuch database for tags of
27 interest to nmbug. If not specified 'notmuch::' is used.
30 from __future__ import print_function
31 from __future__ import unicode_literals
33 import codecs as _codecs
34 import collections as _collections
35 import functools as _functools
36 import inspect as _inspect
37 import locale as _locale
38 import logging as _logging
41 import shutil as _shutil
42 import subprocess as _subprocess
44 import tempfile as _tempfile
45 import textwrap as _textwrap
46 from urllib.parse import quote as _quote
47 from urllib.parse import unquote as _unquote
49 _LOG = _logging.getLogger('nmbug')
50 _LOG.setLevel(_logging.WARNING)
51 _LOG.addHandler(_logging.StreamHandler())
56 _HEX_ESCAPE_REGEX = _re.compile('%[0-9A-F]{2}')
57 _TAG_DIRECTORY = 'tags/'
58 _TAG_FILE_REGEX = _re.compile(_TAG_DIRECTORY + '(?P<id>[^/]*)/(?P<tag>[^/]*)')
60 # magic hash for Git (git hash-object -t blob /dev/null)
61 _EMPTYBLOB = 'e69de29bb2d1d6434b8b29ae775ad8c2e48c5391'
63 def _hex_quote(string, safe='+@=:,'):
65 quote('abc def') -> 'abc%20def'.
67 Wrap urllib.parse.quote with additional safe characters (in
68 addition to letters, digits, and '_.-') and lowercase hex digits
69 (e.g. '%3a' instead of '%3A').
71 uppercase_escapes = _quote(string, safe)
72 return _HEX_ESCAPE_REGEX.sub(
73 lambda match: match.group(0).lower(),
76 def _xapian_quote(string):
78 Quote a string for Xapian's QueryParser.
80 Xapian uses double-quotes for quoting strings. You can escape
81 internal quotes by repeating them [1,2,3].
83 [1]: https://trac.xapian.org/ticket/128#comment:2
84 [2]: https://trac.xapian.org/ticket/128#comment:17
85 [3]: https://trac.xapian.org/changeset/13823/svn
87 return '"{0}"'.format(string.replace('"', '""'))
90 def _xapian_unquote(string):
92 Unquote a Xapian-quoted string.
94 if string.startswith('"') and string.endswith('"'):
95 return string[1:-1].replace('""', '"')
99 class SubprocessError(RuntimeError):
100 "A subprocess exited with a nonzero status"
101 def __init__(self, args, status, stdout=None, stderr=None):
105 msg = '{args} exited with {status}'.format(args=args, status=status)
107 msg = '{msg}: {stderr}'.format(msg=msg, stderr=stderr)
108 super(SubprocessError, self).__init__(msg)
111 class _SubprocessContextManager(object):
113 PEP 343 context manager for subprocesses.
115 'expect' holds a tuple of acceptable exit codes, otherwise we'll
116 raise a SubprocessError in __exit__.
118 def __init__(self, process, args, expect=(0,)):
119 self._process = process
121 self._expect = expect
126 def __exit__(self, type, value, traceback):
127 for name in ['stdin', 'stdout', 'stderr']:
128 stream = getattr(self._process, name)
131 setattr(self._process, name, None)
132 status = self._process.wait()
134 'collect {args} with status {status} (expected {expect})'.format(
135 args=self._args, status=status, expect=self._expect))
136 if status not in self._expect:
137 raise SubprocessError(args=self._args, status=status)
140 return self._process.wait()
143 def _spawn(args, input=None, additional_env=None, wait=False, stdin=None,
144 stdout=None, stderr=None, encoding=_locale.getpreferredencoding(),
145 expect=(0,), **kwargs):
146 """Spawn a subprocess, and optionally wait for it to finish.
148 This wrapper around subprocess.Popen has two modes, depending on
149 the truthiness of 'wait'. If 'wait' is true, we use p.communicate
150 internally to write 'input' to the subprocess's stdin and read
151 from it's stdout/stderr. If 'wait' is False, we return a
152 _SubprocessContextManager instance for fancier handling
153 (e.g. piping between processes).
155 For 'wait' calls when you want to write to the subprocess's stdin,
156 you only need to set 'input' to your content. When 'input' is not
157 None but 'stdin' is, we'll automatically set 'stdin' to PIPE
158 before calling Popen. This avoids having the subprocess
159 accidentally inherit the launching process's stdin.
161 _LOG.debug('spawn {args} (additional env. var.: {env})'.format(
162 args=args, env=additional_env))
163 if not stdin and input is not None:
164 stdin = _subprocess.PIPE
166 if not kwargs.get('env'):
167 kwargs['env'] = dict(_os.environ)
168 kwargs['env'].update(additional_env)
169 p = _subprocess.Popen(
170 args, stdin=stdin, stdout=stdout, stderr=stderr, **kwargs)
172 if hasattr(input, 'encode'):
173 input = input.encode(encoding)
174 (stdout, stderr) = p.communicate(input=input)
177 'collect {args} with status {status} (expected {expect})'.format(
178 args=args, status=status, expect=expect))
179 if stdout is not None:
180 stdout = stdout.decode(encoding)
181 if stderr is not None:
182 stderr = stderr.decode(encoding)
183 if status not in expect:
184 raise SubprocessError(
185 args=args, status=status, stdout=stdout, stderr=stderr)
186 return (status, stdout, stderr)
187 if p.stdin and not stdin:
191 p.stdin = _codecs.getwriter(encoding=encoding)(stream=p.stdin)
192 stream_reader = _codecs.getreader(encoding=encoding)
194 p.stdout = stream_reader(stream=p.stdout)
196 p.stderr = stream_reader(stream=p.stderr)
197 return _SubprocessContextManager(args=args, process=p, expect=expect)
200 def _git(args, **kwargs):
201 args = ['git', '--git-dir', NMBGIT] + list(args)
202 return _spawn(args=args, **kwargs)
205 def _get_current_branch():
206 """Get the name of the current branch.
208 Return 'None' if we're not on a branch.
211 (status, branch, stderr) = _git(
212 args=['symbolic-ref', '--short', 'HEAD'],
213 stdout=_subprocess.PIPE, stderr=_subprocess.PIPE, wait=True)
214 except SubprocessError as e:
215 if 'not a symbolic ref' in e:
218 return branch.strip()
222 "Get the default remote for the current branch."
223 local_branch = _get_current_branch()
224 (status, remote, stderr) = _git(
225 args=['config', 'branch.{0}.remote'.format(local_branch)],
226 stdout=_subprocess.PIPE, wait=True)
227 return remote.strip()
230 def get_tags(prefix=None):
231 "Get a list of tags with a given prefix."
234 (status, stdout, stderr) = _spawn(
235 args=['notmuch', 'search', '--output=tags', '*'],
236 stdout=_subprocess.PIPE, wait=True)
237 return [tag for tag in stdout.splitlines() if tag.startswith(prefix)]
240 def archive(treeish='HEAD', args=()):
242 Dump a tar archive of the current nmbug tag set.
246 Each tag $tag for message with Message-Id $id is written to
249 tags/encode($id)/encode($tag)
251 The encoding preserves alphanumerics, and the characters
252 "+-_@=.:," (not the quotes). All other octets are replaced with
253 '%' followed by a two digit hex number.
255 _git(args=['archive', treeish] + list(args), wait=True)
258 def clone(repository):
260 Create a local nmbug repository from a remote source.
262 This wraps 'git clone', adding some options to avoid creating a
263 working tree while preserving remote-tracking branches and
266 with _tempfile.TemporaryDirectory(prefix='nmbug-clone.') as workdir:
269 'git', 'clone', '--no-checkout', '--separate-git-dir', NMBGIT,
270 repository, workdir],
272 _git(args=['config', '--unset', 'core.worktree'], wait=True, expect=(0, 5))
273 _git(args=['config', 'core.bare', 'true'], wait=True)
274 (status, stdout, stderr) = _git(args=['show-ref', '--verify',
276 'refs/remotes/origin/config'],
280 _git(args=['branch', 'config', 'origin/config'], wait=True)
281 existing_tags = get_tags()
284 'Not checking out to avoid clobbering existing tags: {}'.format(
285 ', '.join(existing_tags)))
290 def _is_committed(status):
291 return len(status['added']) + len(status['deleted']) == 0
294 def commit(treeish='HEAD', message=None):
296 Commit prefix-matching tags from the notmuch database to Git.
298 status = get_status()
300 if _is_committed(status=status):
301 _LOG.warning('Nothing to commit')
304 _git(args=['read-tree', '--empty'], wait=True)
305 _git(args=['read-tree', treeish], wait=True)
307 _update_index(status=status)
310 stdout=_subprocess.PIPE,
312 (_, parent, _) = _git(
313 args=['rev-parse', treeish],
314 stdout=_subprocess.PIPE,
316 (_, commit, _) = _git(
317 args=['commit-tree', tree.strip(), '-p', parent.strip()],
319 stdout=_subprocess.PIPE,
322 args=['update-ref', treeish, commit.strip()],
323 stdout=_subprocess.PIPE,
325 except Exception as e:
326 _git(args=['read-tree', '--empty'], wait=True)
327 _git(args=['read-tree', treeish], wait=True)
330 def _update_index(status):
332 args=['update-index', '--index-info'],
333 stdin=_subprocess.PIPE) as p:
334 for id, tags in status['deleted'].items():
335 for line in _index_tags_for_message(id=id, status='D', tags=tags):
337 for id, tags in status['added'].items():
338 for line in _index_tags_for_message(id=id, status='A', tags=tags):
342 def fetch(remote=None):
344 Fetch changes from the remote repository.
346 See 'merge' to bring those changes into notmuch.
351 _git(args=args, wait=True)
354 def init(remote=None):
356 Create an empty nmbug repository.
358 This wraps 'git init' with a few extra steps to support subsequent
359 status and commit commands.
361 _spawn(args=['git', '--git-dir', NMBGIT, 'init', '--bare'], wait=True)
362 _git(args=['config', 'core.logallrefupdates', 'true'], wait=True)
363 # create an empty blob (e69de29bb2d1d6434b8b29ae775ad8c2e48c5391)
364 _git(args=['hash-object', '-w', '--stdin'], input='', wait=True)
367 'commit', '--allow-empty', '-m', 'Start a new nmbug repository'
369 additional_env={'GIT_WORK_TREE': NMBGIT},
375 Update the notmuch database from Git.
377 This is mainly useful to discard your changes in notmuch relative
380 status = get_status()
382 args=['notmuch', 'tag', '--batch'], stdin=_subprocess.PIPE) as p:
383 for id, tags in status['added'].items():
384 p.stdin.write(_batch_line(action='-', id=id, tags=tags))
385 for id, tags in status['deleted'].items():
386 p.stdin.write(_batch_line(action='+', id=id, tags=tags))
389 def _batch_line(action, id, tags):
391 'notmuch tag --batch' line for adding/removing tags.
393 Set 'action' to '-' to remove a tag or '+' to add the tags to a
396 tag_string = ' '.join(
397 '{action}{prefix}{tag}'.format(
398 action=action, prefix=_ENCODED_TAG_PREFIX, tag=_hex_quote(tag))
400 line = '{tags} -- id:{id}\n'.format(
401 tags=tag_string, id=_xapian_quote(string=id))
405 def _insist_committed():
406 "Die if the the notmuch tags don't match the current HEAD."
407 status = get_status()
408 if not _is_committed(status=status):
409 _LOG.error('\n'.join([
410 'Uncommitted changes to {prefix}* tags in notmuch',
412 "For a summary of changes, run 'nmbug status'",
413 "To save your changes, run 'nmbug commit' before merging/pull",
414 "To discard your changes, run 'nmbug checkout'",
415 ]).format(prefix=TAG_PREFIX))
419 def pull(repository=None, refspecs=None):
421 Pull (merge) remote repository changes to notmuch.
423 'pull' is equivalent to 'fetch' followed by 'merge'. We use the
424 Git-configured repository for your current branch
425 (branch.<name>.repository, likely 'origin', and
426 branch.<name>.merge, likely 'master').
429 if refspecs and not repository:
430 repository = _get_remote()
433 args.append(repository)
435 args.extend(refspecs)
436 with _tempfile.TemporaryDirectory(prefix='nmbug-pull.') as workdir:
442 additional_env={'GIT_WORK_TREE': workdir},
447 def merge(reference='@{upstream}'):
449 Merge changes from 'reference' into HEAD and load the result into notmuch.
451 The default reference is '@{upstream}'.
454 with _tempfile.TemporaryDirectory(prefix='nmbug-merge.') as workdir:
457 ['merge', reference]]:
460 additional_env={'GIT_WORK_TREE': workdir},
467 A simple wrapper for 'git log'.
469 After running 'nmbug fetch', you can inspect the changes with
470 'nmbug log HEAD..@{upstream}'.
472 # we don't want output trapping here, because we want the pager.
473 args = ['log', '--name-status', '--no-renames'] + list(args)
474 with _git(args=args, expect=(0, 1, -13)) as p:
478 def push(repository=None, refspecs=None):
479 "Push the local nmbug Git state to a remote repository."
480 if refspecs and not repository:
481 repository = _get_remote()
484 args.append(repository)
486 args.extend(refspecs)
487 _git(args=args, wait=True)
492 Show pending updates in notmuch or git repo.
494 Prints lines of the form
498 where n is a single character representing notmuch database status
502 Tag is present in notmuch database, but not committed to nmbug
503 (equivalently, tag has been deleted in nmbug repo, e.g. by a
504 pull, but not restored to notmuch database).
508 Tag is present in nmbug repo, but not restored to notmuch
509 database (equivalently, tag has been deleted in notmuch).
513 Message is unknown (missing from local notmuch database).
515 The second character (if present) represents a difference between
516 local and upstream branches. Typically 'nmbug fetch' needs to be
521 Tag is present in upstream, but not in the local Git branch.
525 Tag is present in local Git branch, but not upstream.
527 status = get_status()
528 # 'output' is a nested defaultdict for message status:
529 # * The outer dict is keyed by message id.
530 # * The inner dict is keyed by tag name.
531 # * The inner dict values are status strings (' a', 'Dd', ...).
532 output = _collections.defaultdict(
533 lambda : _collections.defaultdict(lambda : ' '))
534 for id, tags in status['added'].items():
536 output[id][tag] = 'A'
537 for id, tags in status['deleted'].items():
539 output[id][tag] = 'D'
540 for id, tags in status['missing'].items():
542 output[id][tag] = 'U'
544 for id, tag in _diff_refs(filter='A'):
545 output[id][tag] += 'a'
546 for id, tag in _diff_refs(filter='D'):
547 output[id][tag] += 'd'
548 for id, tag_status in sorted(output.items()):
549 for tag, status in sorted(tag_status.items()):
550 print('{status}\t{id}\t{tag}'.format(
551 status=status, id=id, tag=tag))
554 def _is_unmerged(ref='@{upstream}'):
556 (status, fetch_head, stderr) = _git(
557 args=['rev-parse', ref],
558 stdout=_subprocess.PIPE, stderr=_subprocess.PIPE, wait=True)
559 except SubprocessError as e:
560 if 'No upstream configured' in e.stderr:
563 (status, base, stderr) = _git(
564 args=['merge-base', 'HEAD', ref],
565 stdout=_subprocess.PIPE, wait=True)
566 return base != fetch_head
574 index = _index_tags()
575 maybe_deleted = _diff_index(index=index, filter='D')
576 for id, tags in maybe_deleted.items():
577 (_, stdout, stderr) = _spawn(
578 args=['notmuch', 'search', '--output=files', 'id:{0}'.format(id)],
579 stdout=_subprocess.PIPE,
582 status['deleted'][id] = tags
584 status['missing'][id] = tags
585 status['added'] = _diff_index(index=index, filter='A')
591 "Write notmuch tags to the nmbug.index."
592 path = _os.path.join(NMBGIT, 'nmbug.index')
593 query = ' '.join('tag:"{tag}"'.format(tag=tag) for tag in get_tags())
594 prefix = '+{0}'.format(_ENCODED_TAG_PREFIX)
596 args=['read-tree', '--empty'],
597 additional_env={'GIT_INDEX_FILE': path}, wait=True)
599 args=['notmuch', 'dump', '--format=batch-tag', '--', query],
600 stdout=_subprocess.PIPE) as notmuch:
602 args=['update-index', '--index-info'],
603 stdin=_subprocess.PIPE,
604 additional_env={'GIT_INDEX_FILE': path}) as git:
605 for line in notmuch.stdout:
606 if line.strip().startswith('#'):
608 (tags_string, id) = [_.strip() for _ in line.split(' -- id:')]
610 _unquote(tag[len(prefix):])
611 for tag in tags_string.split()
612 if tag.startswith(prefix)]
613 id = _xapian_unquote(string=id)
614 for line in _index_tags_for_message(
615 id=id, status='A', tags=tags):
616 git.stdin.write(line)
620 def _index_tags_for_message(id, status, tags):
622 Update the Git index to either create or delete an empty file.
624 Neither 'id' nor the tags in 'tags' should be encoded/escaped.
631 hash = '0000000000000000000000000000000000000000'
634 path = 'tags/{id}/{tag}'.format(
635 id=_hex_quote(string=id), tag=_hex_quote(string=tag))
636 yield '{mode} {hash}\t{path}\n'.format(mode=mode, hash=hash, path=path)
639 def _diff_index(index, filter):
641 Get an {id: {tag, ...}} dict for a given filter.
643 For example, use 'A' to find added tags, and 'D' to find deleted tags.
645 s = _collections.defaultdict(set)
648 'diff-index', '--cached', '--diff-filter', filter,
649 '--name-only', 'HEAD'],
650 additional_env={'GIT_INDEX_FILE': index},
651 stdout=_subprocess.PIPE) as p:
652 # Once we drop Python < 3.3, we can use 'yield from' here
653 for id, tag in _unpack_diff_lines(stream=p.stdout):
658 def _diff_refs(filter, a='HEAD', b='@{upstream}'):
660 args=['diff', '--diff-filter', filter, '--name-only', a, b],
661 stdout=_subprocess.PIPE) as p:
662 # Once we drop Python < 3.3, we can use 'yield from' here
663 for id, tag in _unpack_diff_lines(stream=p.stdout):
667 def _unpack_diff_lines(stream):
668 "Iterate through (id, tag) tuples in a diff stream."
670 match = _TAG_FILE_REGEX.match(line.strip())
672 message = 'non-tag line in diff: {!r}'.format(line.strip())
673 if line.startswith(_TAG_DIRECTORY):
674 raise ValueError(message)
677 id = _unquote(match.group('id'))
678 tag = _unquote(match.group('tag'))
682 def _help(parser, command=None):
684 Show help for an nmbug command.
686 Because some folks prefer:
692 $ nmbug COMMAND --help
695 parser.parse_args([command, '--help'])
697 parser.parse_args(['--help'])
700 if __name__ == '__main__':
703 parser = argparse.ArgumentParser(
704 description=__doc__.strip(),
705 formatter_class=argparse.RawDescriptionHelpFormatter)
707 '-C', '--git-dir', metavar='REPO',
708 help='Git repository to operate on.')
710 '-p', '--tag-prefix', metavar='PREFIX',
711 default = _os.getenv('NMBPREFIX', 'notmuch::'),
712 help='Prefix of tags to operate on.')
715 choices=['critical', 'error', 'warning', 'info', 'debug'],
716 help='Log verbosity. Defaults to {!r}.'.format(
717 _logging.getLevelName(_LOG.level).lower()))
719 help = _functools.partial(_help, parser=parser)
720 help.__doc__ = _help.__doc__
721 subparsers = parser.add_subparsers(
724 'For help on a particular command, run: '
725 "'%(prog)s ... <command> --help'."))
740 func = locals()[command]
741 doc = _textwrap.dedent(func.__doc__).strip().replace('%', '%%')
742 subparser = subparsers.add_parser(
744 help=doc.splitlines()[0],
746 formatter_class=argparse.RawDescriptionHelpFormatter)
747 subparser.set_defaults(func=func)
748 if command == 'archive':
749 subparser.add_argument(
750 'treeish', metavar='TREE-ISH', nargs='?', default='HEAD',
752 'The tree or commit to produce an archive for. Defaults '
754 subparser.add_argument(
755 'args', metavar='ARG', nargs='*',
757 "Argument passed through to 'git archive'. Set anything "
758 'before <tree-ish>, see git-archive(1) for details.'))
759 elif command == 'clone':
760 subparser.add_argument(
763 'The (possibly remote) repository to clone from. See the '
764 'URLS section of git-clone(1) for more information on '
765 'specifying repositories.'))
766 elif command == 'commit':
767 subparser.add_argument(
768 'message', metavar='MESSAGE', default='', nargs='?',
769 help='Text for the commit message.')
770 elif command == 'fetch':
771 subparser.add_argument(
772 'remote', metavar='REMOTE', nargs='?',
774 'Override the default configured in branch.<name>.remote '
775 'to fetch from a particular remote repository (e.g. '
777 elif command == 'help':
778 subparser.add_argument(
779 'command', metavar='COMMAND', nargs='?',
780 help='The command to show help for.')
781 elif command == 'log':
782 subparser.add_argument(
783 'args', metavar='ARG', nargs='*',
784 help="Additional argument passed through to 'git log'.")
785 elif command == 'merge':
786 subparser.add_argument(
787 'reference', metavar='REFERENCE', default='@{upstream}',
790 'Reference, usually other branch heads, to merge into '
791 "our branch. Defaults to '@{upstream}'."))
792 elif command == 'pull':
793 subparser.add_argument(
794 'repository', metavar='REPOSITORY', default=None, nargs='?',
796 'The "remote" repository that is the source of the pull. '
797 'This parameter can be either a URL (see the section GIT '
798 'URLS in git-pull(1)) or the name of a remote (see the '
799 'section REMOTES in git-pull(1)).'))
800 subparser.add_argument(
801 'refspecs', metavar='REFSPEC', default=None, nargs='*',
803 'Refspec (usually a branch name) to fetch and merge. See '
804 'the <refspec> entry in the OPTIONS section of '
805 'git-pull(1) for other possibilities.'))
806 elif command == 'push':
807 subparser.add_argument(
808 'repository', metavar='REPOSITORY', default=None, nargs='?',
810 'The "remote" repository that is the destination of the '
811 'push. This parameter can be either a URL (see the '
812 'section GIT URLS in git-push(1)) or the name of a remote '
813 '(see the section REMOTES in git-push(1)).'))
814 subparser.add_argument(
815 'refspecs', metavar='REFSPEC', default=None, nargs='*',
817 'Refspec (usually a branch name) to push. See '
818 'the <refspec> entry in the OPTIONS section of '
819 'git-push(1) for other possibilities.'))
821 args = parser.parse_args()
824 NMBGIT = args.git_dir
826 NMBGIT = _os.path.expanduser(
827 _os.getenv('NMBGIT', _os.path.join('~', '.nmbug')))
828 _NMBGIT = _os.path.join(NMBGIT, '.git')
829 if _os.path.isdir(_NMBGIT):
832 TAG_PREFIX = args.tag_prefix
833 _ENCODED_TAG_PREFIX = _hex_quote(TAG_PREFIX, safe='+@=,') # quote ':'
836 level = getattr(_logging, args.log_level.upper())
839 if not getattr(args, 'func', None):
843 if args.func == help:
844 arg_names = ['command']
846 (arg_names, varargs, varkw) = _inspect.getargs(args.func.__code__)
847 kwargs = {key: getattr(args, key) for key in arg_names if key in args}
850 except SubprocessError as e:
851 if _LOG.level == _logging.DEBUG:
852 raise # don't mask the traceback