3 # Copyright (c) 2011-2014 David Bremner <david@tethera.net>
4 # W. Trevor King <wking@tremily.us>
6 # This program is free software: you can redistribute it and/or modify
7 # it under the terms of the GNU General Public License as published by
8 # the Free Software Foundation, either version 3 of the License, or
9 # (at your option) any later version.
11 # This program is distributed in the hope that it will be useful,
12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 # GNU General Public License for more details.
16 # You should have received a copy of the GNU General Public License
17 # along with this program. If not, see https://www.gnu.org/licenses/ .
20 Manage notmuch tags with Git
22 Environment variables:
24 * NMBGIT specifies the location of the git repository used by nmbug.
25 If not specified $HOME/.nmbug is used.
26 * NMBPREFIX specifies the prefix in the notmuch database for tags of
27 interest to nmbug. If not specified 'notmuch::' is used.
30 from __future__ import print_function
31 from __future__ import unicode_literals
33 import codecs as _codecs
34 import collections as _collections
35 import functools as _functools
36 import inspect as _inspect
37 import locale as _locale
38 import logging as _logging
41 import shutil as _shutil
42 import subprocess as _subprocess
44 import tempfile as _tempfile
45 import textwrap as _textwrap
46 from urllib.parse import quote as _quote
47 from urllib.parse import unquote as _unquote
49 _LOG = _logging.getLogger('nmbug')
50 _LOG.setLevel(_logging.WARNING)
51 _LOG.addHandler(_logging.StreamHandler())
56 _HEX_ESCAPE_REGEX = _re.compile('%[0-9A-F]{2}')
57 _TAG_DIRECTORY = 'tags/'
58 _TAG_FILE_REGEX = _re.compile(_TAG_DIRECTORY + '(?P<id>[^/]*)/(?P<tag>[^/]*)')
60 # magic hash for Git (git hash-object -t blob /dev/null)
61 _EMPTYBLOB = 'e69de29bb2d1d6434b8b29ae775ad8c2e48c5391'
63 def _hex_quote(string, safe='+@=:,'):
65 quote('abc def') -> 'abc%20def'.
67 Wrap urllib.parse.quote with additional safe characters (in
68 addition to letters, digits, and '_.-') and lowercase hex digits
69 (e.g. '%3a' instead of '%3A').
71 uppercase_escapes = _quote(string, safe)
72 return _HEX_ESCAPE_REGEX.sub(
73 lambda match: match.group(0).lower(),
76 def _xapian_quote(string):
78 Quote a string for Xapian's QueryParser.
80 Xapian uses double-quotes for quoting strings. You can escape
81 internal quotes by repeating them [1,2,3].
83 [1]: https://trac.xapian.org/ticket/128#comment:2
84 [2]: https://trac.xapian.org/ticket/128#comment:17
85 [3]: https://trac.xapian.org/changeset/13823/svn
87 return '"{0}"'.format(string.replace('"', '""'))
90 def _xapian_unquote(string):
92 Unquote a Xapian-quoted string.
94 if string.startswith('"') and string.endswith('"'):
95 return string[1:-1].replace('""', '"')
99 class SubprocessError(RuntimeError):
100 "A subprocess exited with a nonzero status"
101 def __init__(self, args, status, stdout=None, stderr=None):
105 msg = '{args} exited with {status}'.format(args=args, status=status)
107 msg = '{msg}: {stderr}'.format(msg=msg, stderr=stderr)
108 super(SubprocessError, self).__init__(msg)
111 class _SubprocessContextManager(object):
113 PEP 343 context manager for subprocesses.
115 'expect' holds a tuple of acceptable exit codes, otherwise we'll
116 raise a SubprocessError in __exit__.
118 def __init__(self, process, args, expect=(0,)):
119 self._process = process
121 self._expect = expect
126 def __exit__(self, type, value, traceback):
127 for name in ['stdin', 'stdout', 'stderr']:
128 stream = getattr(self._process, name)
131 setattr(self._process, name, None)
132 status = self._process.wait()
134 'collect {args} with status {status} (expected {expect})'.format(
135 args=self._args, status=status, expect=self._expect))
136 if status not in self._expect:
137 raise SubprocessError(args=self._args, status=status)
140 return self._process.wait()
143 def _spawn(args, input=None, additional_env=None, wait=False, stdin=None,
144 stdout=None, stderr=None, encoding=_locale.getpreferredencoding(),
145 expect=(0,), **kwargs):
146 """Spawn a subprocess, and optionally wait for it to finish.
148 This wrapper around subprocess.Popen has two modes, depending on
149 the truthiness of 'wait'. If 'wait' is true, we use p.communicate
150 internally to write 'input' to the subprocess's stdin and read
151 from it's stdout/stderr. If 'wait' is False, we return a
152 _SubprocessContextManager instance for fancier handling
153 (e.g. piping between processes).
155 For 'wait' calls when you want to write to the subprocess's stdin,
156 you only need to set 'input' to your content. When 'input' is not
157 None but 'stdin' is, we'll automatically set 'stdin' to PIPE
158 before calling Popen. This avoids having the subprocess
159 accidentally inherit the launching process's stdin.
161 _LOG.debug('spawn {args} (additional env. var.: {env})'.format(
162 args=args, env=additional_env))
163 if not stdin and input is not None:
164 stdin = _subprocess.PIPE
166 if not kwargs.get('env'):
167 kwargs['env'] = dict(_os.environ)
168 kwargs['env'].update(additional_env)
169 p = _subprocess.Popen(
170 args, stdin=stdin, stdout=stdout, stderr=stderr, **kwargs)
172 if hasattr(input, 'encode'):
173 input = input.encode(encoding)
174 (stdout, stderr) = p.communicate(input=input)
177 'collect {args} with status {status} (expected {expect})'.format(
178 args=args, status=status, expect=expect))
179 if stdout is not None:
180 stdout = stdout.decode(encoding)
181 if stderr is not None:
182 stderr = stderr.decode(encoding)
183 if status not in expect:
184 raise SubprocessError(
185 args=args, status=status, stdout=stdout, stderr=stderr)
186 return (status, stdout, stderr)
187 if p.stdin and not stdin:
191 p.stdin = _codecs.getwriter(encoding=encoding)(stream=p.stdin)
192 stream_reader = _codecs.getreader(encoding=encoding)
194 p.stdout = stream_reader(stream=p.stdout)
196 p.stderr = stream_reader(stream=p.stderr)
197 return _SubprocessContextManager(args=args, process=p, expect=expect)
200 def _git(args, **kwargs):
201 args = ['git', '--git-dir', NMBGIT] + list(args)
202 return _spawn(args=args, **kwargs)
205 def _get_current_branch():
206 """Get the name of the current branch.
208 Return 'None' if we're not on a branch.
211 (status, branch, stderr) = _git(
212 args=['symbolic-ref', '--short', 'HEAD'],
213 stdout=_subprocess.PIPE, stderr=_subprocess.PIPE, wait=True)
214 except SubprocessError as e:
215 if 'not a symbolic ref' in e:
218 return branch.strip()
222 "Get the default remote for the current branch."
223 local_branch = _get_current_branch()
224 (status, remote, stderr) = _git(
225 args=['config', 'branch.{0}.remote'.format(local_branch)],
226 stdout=_subprocess.PIPE, wait=True)
227 return remote.strip()
230 def get_tags(prefix=None):
231 "Get a list of tags with a given prefix."
234 (status, stdout, stderr) = _spawn(
235 args=['notmuch', 'search', '--output=tags', '*'],
236 stdout=_subprocess.PIPE, wait=True)
237 return [tag for tag in stdout.splitlines() if tag.startswith(prefix)]
240 def archive(treeish='HEAD', args=()):
242 Dump a tar archive of the current nmbug tag set.
246 Each tag $tag for message with Message-Id $id is written to
249 tags/encode($id)/encode($tag)
251 The encoding preserves alphanumerics, and the characters
252 "+-_@=.:," (not the quotes). All other octets are replaced with
253 '%' followed by a two digit hex number.
255 _git(args=['archive', treeish] + list(args), wait=True)
258 def clone(repository):
260 Create a local nmbug repository from a remote source.
262 This wraps 'git clone', adding some options to avoid creating a
263 working tree while preserving remote-tracking branches and
266 with _tempfile.TemporaryDirectory(prefix='nmbug-clone.') as workdir:
269 'git', 'clone', '--no-checkout', '--separate-git-dir', NMBGIT,
270 repository, workdir],
272 _git(args=['config', '--unset', 'core.worktree'], wait=True, expect=(0, 5))
273 _git(args=['config', 'core.bare', 'true'], wait=True)
274 _git(args=['branch', 'config', 'origin/config'], wait=True)
275 existing_tags = get_tags()
278 'Not checking out to avoid clobbering existing tags: {}'.format(
279 ', '.join(existing_tags)))
284 def _is_committed(status):
285 return len(status['added']) + len(status['deleted']) == 0
288 def commit(treeish='HEAD', message=None):
290 Commit prefix-matching tags from the notmuch database to Git.
292 status = get_status()
294 if _is_committed(status=status):
295 _LOG.warning('Nothing to commit')
298 _git(args=['read-tree', '--empty'], wait=True)
299 _git(args=['read-tree', treeish], wait=True)
301 _update_index(status=status)
304 stdout=_subprocess.PIPE,
306 (_, parent, _) = _git(
307 args=['rev-parse', treeish],
308 stdout=_subprocess.PIPE,
310 (_, commit, _) = _git(
311 args=['commit-tree', tree.strip(), '-p', parent.strip()],
313 stdout=_subprocess.PIPE,
316 args=['update-ref', treeish, commit.strip()],
317 stdout=_subprocess.PIPE,
319 except Exception as e:
320 _git(args=['read-tree', '--empty'], wait=True)
321 _git(args=['read-tree', treeish], wait=True)
324 def _update_index(status):
326 args=['update-index', '--index-info'],
327 stdin=_subprocess.PIPE) as p:
328 for id, tags in status['deleted'].items():
329 for line in _index_tags_for_message(id=id, status='D', tags=tags):
331 for id, tags in status['added'].items():
332 for line in _index_tags_for_message(id=id, status='A', tags=tags):
336 def fetch(remote=None):
338 Fetch changes from the remote repository.
340 See 'merge' to bring those changes into notmuch.
345 _git(args=args, wait=True)
350 Update the notmuch database from Git.
352 This is mainly useful to discard your changes in notmuch relative
355 status = get_status()
357 args=['notmuch', 'tag', '--batch'], stdin=_subprocess.PIPE) as p:
358 for id, tags in status['added'].items():
359 p.stdin.write(_batch_line(action='-', id=id, tags=tags))
360 for id, tags in status['deleted'].items():
361 p.stdin.write(_batch_line(action='+', id=id, tags=tags))
364 def _batch_line(action, id, tags):
366 'notmuch tag --batch' line for adding/removing tags.
368 Set 'action' to '-' to remove a tag or '+' to add the tags to a
371 tag_string = ' '.join(
372 '{action}{prefix}{tag}'.format(
373 action=action, prefix=_ENCODED_TAG_PREFIX, tag=_hex_quote(tag))
375 line = '{tags} -- id:{id}\n'.format(
376 tags=tag_string, id=_xapian_quote(string=id))
380 def _insist_committed():
381 "Die if the the notmuch tags don't match the current HEAD."
382 status = get_status()
383 if not _is_committed(status=status):
384 _LOG.error('\n'.join([
385 'Uncommitted changes to {prefix}* tags in notmuch',
387 "For a summary of changes, run 'nmbug status'",
388 "To save your changes, run 'nmbug commit' before merging/pull",
389 "To discard your changes, run 'nmbug checkout'",
390 ]).format(prefix=TAG_PREFIX))
394 def pull(repository=None, refspecs=None):
396 Pull (merge) remote repository changes to notmuch.
398 'pull' is equivalent to 'fetch' followed by 'merge'. We use the
399 Git-configured repository for your current branch
400 (branch.<name>.repository, likely 'origin', and
401 branch.<name>.merge, likely 'master').
404 if refspecs and not repository:
405 repository = _get_remote()
408 args.append(repository)
410 args.extend(refspecs)
411 with _tempfile.TemporaryDirectory(prefix='nmbug-pull.') as workdir:
417 additional_env={'GIT_WORK_TREE': workdir},
422 def merge(reference='@{upstream}'):
424 Merge changes from 'reference' into HEAD and load the result into notmuch.
426 The default reference is '@{upstream}'.
429 with _tempfile.TemporaryDirectory(prefix='nmbug-merge.') as workdir:
432 ['merge', reference]]:
435 additional_env={'GIT_WORK_TREE': workdir},
442 A simple wrapper for 'git log'.
444 After running 'nmbug fetch', you can inspect the changes with
445 'nmbug log HEAD..@{upstream}'.
447 # we don't want output trapping here, because we want the pager.
448 args = ['log', '--name-status', '--no-renames'] + list(args)
449 with _git(args=args, expect=(0, 1, -13)) as p:
453 def push(repository=None, refspecs=None):
454 "Push the local nmbug Git state to a remote repository."
455 if refspecs and not repository:
456 repository = _get_remote()
459 args.append(repository)
461 args.extend(refspecs)
462 _git(args=args, wait=True)
467 Show pending updates in notmuch or git repo.
469 Prints lines of the form
473 where n is a single character representing notmuch database status
477 Tag is present in notmuch database, but not committed to nmbug
478 (equivalently, tag has been deleted in nmbug repo, e.g. by a
479 pull, but not restored to notmuch database).
483 Tag is present in nmbug repo, but not restored to notmuch
484 database (equivalently, tag has been deleted in notmuch).
488 Message is unknown (missing from local notmuch database).
490 The second character (if present) represents a difference between
491 local and upstream branches. Typically 'nmbug fetch' needs to be
496 Tag is present in upstream, but not in the local Git branch.
500 Tag is present in local Git branch, but not upstream.
502 status = get_status()
503 # 'output' is a nested defaultdict for message status:
504 # * The outer dict is keyed by message id.
505 # * The inner dict is keyed by tag name.
506 # * The inner dict values are status strings (' a', 'Dd', ...).
507 output = _collections.defaultdict(
508 lambda : _collections.defaultdict(lambda : ' '))
509 for id, tags in status['added'].items():
511 output[id][tag] = 'A'
512 for id, tags in status['deleted'].items():
514 output[id][tag] = 'D'
515 for id, tags in status['missing'].items():
517 output[id][tag] = 'U'
519 for id, tag in _diff_refs(filter='A'):
520 output[id][tag] += 'a'
521 for id, tag in _diff_refs(filter='D'):
522 output[id][tag] += 'd'
523 for id, tag_status in sorted(output.items()):
524 for tag, status in sorted(tag_status.items()):
525 print('{status}\t{id}\t{tag}'.format(
526 status=status, id=id, tag=tag))
529 def _is_unmerged(ref='@{upstream}'):
531 (status, fetch_head, stderr) = _git(
532 args=['rev-parse', ref],
533 stdout=_subprocess.PIPE, stderr=_subprocess.PIPE, wait=True)
534 except SubprocessError as e:
535 if 'No upstream configured' in e.stderr:
538 (status, base, stderr) = _git(
539 args=['merge-base', 'HEAD', ref],
540 stdout=_subprocess.PIPE, wait=True)
541 return base != fetch_head
549 index = _index_tags()
550 maybe_deleted = _diff_index(index=index, filter='D')
551 for id, tags in maybe_deleted.items():
552 (_, stdout, stderr) = _spawn(
553 args=['notmuch', 'search', '--output=files', 'id:{0}'.format(id)],
554 stdout=_subprocess.PIPE,
557 status['deleted'][id] = tags
559 status['missing'][id] = tags
560 status['added'] = _diff_index(index=index, filter='A')
566 "Write notmuch tags to the nmbug.index."
567 path = _os.path.join(NMBGIT, 'nmbug.index')
568 query = ' '.join('tag:"{tag}"'.format(tag=tag) for tag in get_tags())
569 prefix = '+{0}'.format(_ENCODED_TAG_PREFIX)
571 args=['read-tree', '--empty'],
572 additional_env={'GIT_INDEX_FILE': path}, wait=True)
574 args=['notmuch', 'dump', '--format=batch-tag', '--', query],
575 stdout=_subprocess.PIPE) as notmuch:
577 args=['update-index', '--index-info'],
578 stdin=_subprocess.PIPE,
579 additional_env={'GIT_INDEX_FILE': path}) as git:
580 for line in notmuch.stdout:
581 if line.strip().startswith('#'):
583 (tags_string, id) = [_.strip() for _ in line.split(' -- id:')]
585 _unquote(tag[len(prefix):])
586 for tag in tags_string.split()
587 if tag.startswith(prefix)]
588 id = _xapian_unquote(string=id)
589 for line in _index_tags_for_message(
590 id=id, status='A', tags=tags):
591 git.stdin.write(line)
595 def _index_tags_for_message(id, status, tags):
597 Update the Git index to either create or delete an empty file.
599 Neither 'id' nor the tags in 'tags' should be encoded/escaped.
606 hash = '0000000000000000000000000000000000000000'
609 path = 'tags/{id}/{tag}'.format(
610 id=_hex_quote(string=id), tag=_hex_quote(string=tag))
611 yield '{mode} {hash}\t{path}\n'.format(mode=mode, hash=hash, path=path)
614 def _diff_index(index, filter):
616 Get an {id: {tag, ...}} dict for a given filter.
618 For example, use 'A' to find added tags, and 'D' to find deleted tags.
620 s = _collections.defaultdict(set)
623 'diff-index', '--cached', '--diff-filter', filter,
624 '--name-only', 'HEAD'],
625 additional_env={'GIT_INDEX_FILE': index},
626 stdout=_subprocess.PIPE) as p:
627 # Once we drop Python < 3.3, we can use 'yield from' here
628 for id, tag in _unpack_diff_lines(stream=p.stdout):
633 def _diff_refs(filter, a='HEAD', b='@{upstream}'):
635 args=['diff', '--diff-filter', filter, '--name-only', a, b],
636 stdout=_subprocess.PIPE) as p:
637 # Once we drop Python < 3.3, we can use 'yield from' here
638 for id, tag in _unpack_diff_lines(stream=p.stdout):
642 def _unpack_diff_lines(stream):
643 "Iterate through (id, tag) tuples in a diff stream."
645 match = _TAG_FILE_REGEX.match(line.strip())
647 message = 'non-tag line in diff: {!r}'.format(line.strip())
648 if line.startswith(_TAG_DIRECTORY):
649 raise ValueError(message)
652 id = _unquote(match.group('id'))
653 tag = _unquote(match.group('tag'))
657 def _help(parser, command=None):
659 Show help for an nmbug command.
661 Because some folks prefer:
667 $ nmbug COMMAND --help
670 parser.parse_args([command, '--help'])
672 parser.parse_args(['--help'])
675 if __name__ == '__main__':
678 parser = argparse.ArgumentParser(
679 description=__doc__.strip(),
680 formatter_class=argparse.RawDescriptionHelpFormatter)
682 '-C', '--git-dir', metavar='REPO',
683 help='Git repository to operate on.')
685 '-p', '--tag-prefix', metavar='PREFIX',
686 default = _os.getenv('NMBPREFIX', 'notmuch::'),
687 help='Prefix of tags to operate on.')
690 choices=['critical', 'error', 'warning', 'info', 'debug'],
691 help='Log verbosity. Defaults to {!r}.'.format(
692 _logging.getLevelName(_LOG.level).lower()))
694 help = _functools.partial(_help, parser=parser)
695 help.__doc__ = _help.__doc__
696 subparsers = parser.add_subparsers(
699 'For help on a particular command, run: '
700 "'%(prog)s ... <command> --help'."))
714 func = locals()[command]
715 doc = _textwrap.dedent(func.__doc__).strip().replace('%', '%%')
716 subparser = subparsers.add_parser(
718 help=doc.splitlines()[0],
720 formatter_class=argparse.RawDescriptionHelpFormatter)
721 subparser.set_defaults(func=func)
722 if command == 'archive':
723 subparser.add_argument(
724 'treeish', metavar='TREE-ISH', nargs='?', default='HEAD',
726 'The tree or commit to produce an archive for. Defaults '
728 subparser.add_argument(
729 'args', metavar='ARG', nargs='*',
731 "Argument passed through to 'git archive'. Set anything "
732 'before <tree-ish>, see git-archive(1) for details.'))
733 elif command == 'clone':
734 subparser.add_argument(
737 'The (possibly remote) repository to clone from. See the '
738 'URLS section of git-clone(1) for more information on '
739 'specifying repositories.'))
740 elif command == 'commit':
741 subparser.add_argument(
742 'message', metavar='MESSAGE', default='', nargs='?',
743 help='Text for the commit message.')
744 elif command == 'fetch':
745 subparser.add_argument(
746 'remote', metavar='REMOTE', nargs='?',
748 'Override the default configured in branch.<name>.remote '
749 'to fetch from a particular remote repository (e.g. '
751 elif command == 'help':
752 subparser.add_argument(
753 'command', metavar='COMMAND', nargs='?',
754 help='The command to show help for.')
755 elif command == 'log':
756 subparser.add_argument(
757 'args', metavar='ARG', nargs='*',
758 help="Additional argument passed through to 'git log'.")
759 elif command == 'merge':
760 subparser.add_argument(
761 'reference', metavar='REFERENCE', default='@{upstream}',
764 'Reference, usually other branch heads, to merge into '
765 "our branch. Defaults to '@{upstream}'."))
766 elif command == 'pull':
767 subparser.add_argument(
768 'repository', metavar='REPOSITORY', default=None, nargs='?',
770 'The "remote" repository that is the source of the pull. '
771 'This parameter can be either a URL (see the section GIT '
772 'URLS in git-pull(1)) or the name of a remote (see the '
773 'section REMOTES in git-pull(1)).'))
774 subparser.add_argument(
775 'refspecs', metavar='REFSPEC', default=None, nargs='*',
777 'Refspec (usually a branch name) to fetch and merge. See '
778 'the <refspec> entry in the OPTIONS section of '
779 'git-pull(1) for other possibilities.'))
780 elif command == 'push':
781 subparser.add_argument(
782 'repository', metavar='REPOSITORY', default=None, nargs='?',
784 'The "remote" repository that is the destination of the '
785 'push. This parameter can be either a URL (see the '
786 'section GIT URLS in git-push(1)) or the name of a remote '
787 '(see the section REMOTES in git-push(1)).'))
788 subparser.add_argument(
789 'refspecs', metavar='REFSPEC', default=None, nargs='*',
791 'Refspec (usually a branch name) to push. See '
792 'the <refspec> entry in the OPTIONS section of '
793 'git-push(1) for other possibilities.'))
795 args = parser.parse_args()
798 NMBGIT = args.git_dir
800 NMBGIT = _os.path.expanduser(
801 _os.getenv('NMBGIT', _os.path.join('~', '.nmbug')))
802 _NMBGIT = _os.path.join(NMBGIT, '.git')
803 if _os.path.isdir(_NMBGIT):
806 TAG_PREFIX = args.tag_prefix
807 _ENCODED_TAG_PREFIX = _hex_quote(TAG_PREFIX, safe='+@=,') # quote ':'
810 level = getattr(_logging, args.log_level.upper())
813 if not getattr(args, 'func', None):
817 if args.func == help:
818 arg_names = ['command']
820 (arg_names, varargs, varkw) = _inspect.getargs(args.func.__code__)
821 kwargs = {key: getattr(args, key) for key in arg_names if key in args}
824 except SubprocessError as e:
825 if _LOG.level == _logging.DEBUG:
826 raise # don't mask the traceback