3 # Copyright (c) 2011-2014 David Bremner <david@tethera.net>
4 # W. Trevor King <wking@tremily.us>
6 # This program is free software: you can redistribute it and/or modify
7 # it under the terms of the GNU General Public License as published by
8 # the Free Software Foundation, either version 3 of the License, or
9 # (at your option) any later version.
11 # This program is distributed in the hope that it will be useful,
12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 # GNU General Public License for more details.
16 # You should have received a copy of the GNU General Public License
17 # along with this program. If not, see http://www.gnu.org/licenses/ .
20 Manage notmuch tags with Git
22 Environment variables:
24 * NMBGIT specifies the location of the git repository used by nmbug.
25 If not specified $HOME/.nmbug is used.
26 * NMBPREFIX specifies the prefix in the notmuch database for tags of
27 interest to nmbug. If not specified 'notmuch::' is used.
30 from __future__ import print_function
31 from __future__ import unicode_literals
33 import codecs as _codecs
34 import collections as _collections
35 import functools as _functools
36 import inspect as _inspect
37 import locale as _locale
38 import logging as _logging
41 import shutil as _shutil
42 import subprocess as _subprocess
44 import tempfile as _tempfile
45 import textwrap as _textwrap
47 from urllib.parse import quote as _quote
48 from urllib.parse import unquote as _unquote
49 except ImportError: # Python 2
50 from urllib import quote as _quote
51 from urllib import unquote as _unquote
56 _LOG = _logging.getLogger('nmbug')
57 _LOG.setLevel(_logging.ERROR)
58 _LOG.addHandler(_logging.StreamHandler())
60 NMBGIT = _os.path.expanduser(
61 _os.getenv('NMBGIT', _os.path.join('~', '.nmbug')))
62 _NMBGIT = _os.path.join(NMBGIT, '.git')
63 if _os.path.isdir(_NMBGIT):
66 TAG_PREFIX = _os.getenv('NMBPREFIX', 'notmuch::')
67 _HEX_ESCAPE_REGEX = _re.compile('%[0-9A-F]{2}')
68 _TAG_FILE_REGEX = _re.compile('tags/(?P<id>[^/]*)/(?P<tag>[^/]*)')
70 # magic hash for Git (git hash-object -t blob /dev/null)
71 _EMPTYBLOB = 'e69de29bb2d1d6434b8b29ae775ad8c2e48c5391'
75 getattr(_tempfile, 'TemporaryDirectory')
76 except AttributeError: # Python < 3.2
77 class _TemporaryDirectory(object):
79 Fallback context manager for Python < 3.2
81 See PEP 343 for details on context managers [1].
83 [1]: http://legacy.python.org/dev/peps/pep-0343/
85 def __init__(self, **kwargs):
86 self.name = _tempfile.mkdtemp(**kwargs)
91 def __exit__(self, type, value, traceback):
92 _shutil.rmtree(self.name)
95 _tempfile.TemporaryDirectory = _TemporaryDirectory
98 def _hex_quote(string, safe='+@=:,'):
100 quote('abc def') -> 'abc%20def'.
102 Wrap urllib.parse.quote with additional safe characters (in
103 addition to letters, digits, and '_.-') and lowercase hex digits
104 (e.g. '%3a' instead of '%3A').
106 uppercase_escapes = _quote(string, safe)
107 return _HEX_ESCAPE_REGEX.sub(
108 lambda match: match.group(0).lower(),
112 _ENCODED_TAG_PREFIX = _hex_quote(TAG_PREFIX, safe='+@=,') # quote ':'
115 def _xapian_quote(string):
117 Quote a string for Xapian's QueryParser.
119 Xapian uses double-quotes for quoting strings. You can escape
120 internal quotes by repeating them [1,2,3].
122 [1]: http://trac.xapian.org/ticket/128#comment:2
123 [2]: http://trac.xapian.org/ticket/128#comment:17
124 [3]: http://trac.xapian.org/changeset/13823/svn
126 return '"{0}"'.format(string.replace('"', '""'))
129 def _xapian_unquote(string):
131 Unquote a Xapian-quoted string.
133 if string.startswith('"') and string.endswith('"'):
134 return string[1:-1].replace('""', '"')
138 class SubprocessError(RuntimeError):
139 "A subprocess exited with a nonzero status"
140 def __init__(self, args, status, stdout=None, stderr=None):
144 msg = '{args} exited with {status}'.format(args=args, status=status)
146 msg = '{msg}: {stderr}'.format(msg=msg, stderr=stderr)
147 super(SubprocessError, self).__init__(msg)
150 class _SubprocessContextManager(object):
152 PEP 343 context manager for subprocesses.
154 'expect' holds a tuple of acceptable exit codes, otherwise we'll
155 raise a SubprocessError in __exit__.
157 def __init__(self, process, args, expect=(0,)):
158 self._process = process
160 self._expect = expect
165 def __exit__(self, type, value, traceback):
166 for name in ['stdin', 'stdout', 'stderr']:
167 stream = getattr(self._process, name)
170 setattr(self._process, name, None)
171 status = self._process.wait()
172 _LOG.debug('collect {args} with status {status}'.format(
173 args=self._args, status=status))
174 if status not in self._expect:
175 raise SubprocessError(args=self._args, status=status)
178 return self._process.wait()
181 def _spawn(args, input=None, additional_env=None, wait=False, stdin=None,
182 stdout=None, stderr=None, encoding=_locale.getpreferredencoding(),
183 expect=(0,), **kwargs):
184 """Spawn a subprocess, and optionally wait for it to finish.
186 This wrapper around subprocess.Popen has two modes, depending on
187 the truthiness of 'wait'. If 'wait' is true, we use p.communicate
188 internally to write 'input' to the subprocess's stdin and read
189 from it's stdout/stderr. If 'wait' is False, we return a
190 _SubprocessContextManager instance for fancier handling
191 (e.g. piping between processes).
193 For 'wait' calls when you want to write to the subprocess's stdin,
194 you only need to set 'input' to your content. When 'input' is not
195 None but 'stdin' is, we'll automatically set 'stdin' to PIPE
196 before calling Popen. This avoids having the subprocess
197 accidentally inherit the launching process's stdin.
199 _LOG.debug('spawn {args} (additional env. var.: {env})'.format(
200 args=args, env=additional_env))
201 if not stdin and input is not None:
202 stdin = _subprocess.PIPE
204 if not kwargs.get('env'):
205 kwargs['env'] = dict(_os.environ)
206 kwargs['env'].update(additional_env)
207 p = _subprocess.Popen(
208 args, stdin=stdin, stdout=stdout, stderr=stderr, **kwargs)
210 if hasattr(input, 'encode'):
211 input = input.encode(encoding)
212 (stdout, stderr) = p.communicate(input=input)
214 _LOG.debug('collect {args} with status {status}'.format(
215 args=args, status=status))
216 if stdout is not None:
217 stdout = stdout.decode(encoding)
218 if stderr is not None:
219 stderr = stderr.decode(encoding)
221 raise SubprocessError(
222 args=args, status=status, stdout=stdout, stderr=stderr)
223 return (status, stdout, stderr)
224 if p.stdin and not stdin:
228 p.stdin = _codecs.getwriter(encoding=encoding)(stream=p.stdin)
229 stream_reader = _codecs.getreader(encoding=encoding)
231 p.stdout = stream_reader(stream=p.stdout)
233 p.stderr = stream_reader(stream=p.stderr)
234 return _SubprocessContextManager(args=args, process=p, expect=expect)
237 def _git(args, **kwargs):
238 args = ['git', '--git-dir', NMBGIT] + list(args)
239 return _spawn(args=args, **kwargs)
242 def _get_current_branch():
243 """Get the name of the current branch.
245 Return 'None' if we're not on a branch.
248 (status, branch, stderr) = _git(
249 args=['symbolic-ref', '--short', 'HEAD'],
250 stdout=_subprocess.PIPE, stderr=_subprocess.PIPE, wait=True)
251 except SubprocessError as e:
252 if 'not a symbolic ref' in e:
255 return branch.strip()
259 "Get the default remote for the current branch."
260 local_branch = _get_current_branch()
261 (status, remote, stderr) = _git(
262 args=['config', 'branch.{0}.remote'.format(local_branch)],
263 stdout=_subprocess.PIPE, wait=True)
264 return remote.strip()
267 def get_tags(prefix=None):
268 "Get a list of tags with a given prefix."
271 (status, stdout, stderr) = _spawn(
272 args=['notmuch', 'search', '--output=tags', '*'],
273 stdout=_subprocess.PIPE, wait=True)
274 return [tag for tag in stdout.splitlines() if tag.startswith(prefix)]
277 def archive(treeish='HEAD', args=()):
279 Dump a tar archive of the current nmbug tag set.
283 Each tag $tag for message with Message-Id $id is written to
286 tags/encode($id)/encode($tag)
288 The encoding preserves alphanumerics, and the characters
289 "+-_@=.:," (not the quotes). All other octets are replaced with
290 '%' followed by a two digit hex number.
292 _git(args=['archive', treeish] + list(args), wait=True)
295 def clone(repository):
297 Create a local nmbug repository from a remote source.
299 This wraps 'git clone', adding some options to avoid creating a
300 working tree while preserving remote-tracking branches and
303 with _tempfile.TemporaryDirectory(prefix='nmbug-clone.') as workdir:
306 'git', 'clone', '--no-checkout', '--separate-git-dir', NMBGIT,
307 repository, workdir],
309 _git(args=['config', '--unset', 'core.worktree'], wait=True)
310 _git(args=['config', 'core.bare', 'true'], wait=True)
313 def _is_committed(status):
314 return len(status['added']) + len(status['deleted']) == 0
317 def commit(treeish='HEAD', message=None):
319 Commit prefix-matching tags from the notmuch database to Git.
321 status = get_status()
323 if _is_committed(status=status):
324 _LOG.warning('Nothing to commit')
327 _git(args=['read-tree', '--empty'], wait=True)
328 _git(args=['read-tree', treeish], wait=True)
330 _update_index(status=status)
333 stdout=_subprocess.PIPE,
335 (_, parent, _) = _git(
336 args=['rev-parse', treeish],
337 stdout=_subprocess.PIPE,
339 (_, commit, _) = _git(
340 args=['commit-tree', tree.strip(), '-p', parent.strip()],
342 stdout=_subprocess.PIPE,
345 args=['update-ref', treeish, commit.strip()],
346 stdout=_subprocess.PIPE,
348 except Exception as e:
349 _git(args=['read-tree', '--empty'], wait=True)
350 _git(args=['read-tree', treeish], wait=True)
353 def _update_index(status):
355 args=['update-index', '--index-info'],
356 stdin=_subprocess.PIPE) as p:
357 for id, tags in status['deleted'].items():
358 for line in _index_tags_for_message(id=id, status='D', tags=tags):
360 for id, tags in status['added'].items():
361 for line in _index_tags_for_message(id=id, status='A', tags=tags):
365 def fetch(remote=None):
367 Fetch changes from the remote repository.
369 See 'merge' to bring those changes into notmuch.
374 _git(args=args, wait=True)
379 Update the notmuch database from Git.
381 This is mainly useful to discard your changes in notmuch relative
384 status = get_status()
386 args=['notmuch', 'tag', '--batch'], stdin=_subprocess.PIPE) as p:
387 for id, tags in status['added'].items():
388 p.stdin.write(_batch_line(action='-', id=id, tags=tags))
389 for id, tags in status['deleted'].items():
390 p.stdin.write(_batch_line(action='+', id=id, tags=tags))
393 def _batch_line(action, id, tags):
395 'notmuch tag --batch' line for adding/removing tags.
397 Set 'action' to '-' to remove a tag or '+' to add the tags to a
400 tag_string = ' '.join(
401 '{action}{prefix}{tag}'.format(
402 action=action, prefix=_ENCODED_TAG_PREFIX, tag=_hex_quote(tag))
404 line = '{tags} -- id:{id}\n'.format(
405 tags=tag_string, id=_xapian_quote(string=id))
409 def _insist_committed():
410 "Die if the the notmuch tags don't match the current HEAD."
411 status = get_status()
412 if not _is_committed(status=status):
413 _LOG.error('\n'.join([
414 'Uncommitted changes to {prefix}* tags in notmuch',
416 "For a summary of changes, run 'nmbug status'",
417 "To save your changes, run 'nmbug commit' before merging/pull",
418 "To discard your changes, run 'nmbug checkout'",
419 ]).format(prefix=TAG_PREFIX))
423 def pull(repository=None, refspecs=None):
425 Pull (merge) remote repository changes to notmuch.
427 'pull' is equivalent to 'fetch' followed by 'merge'. We use the
428 Git-configured repository for your current branch
429 (branch.<name>.repository, likely 'origin', and
430 branch.<name>.merge, likely 'master').
433 if refspecs and not repository:
434 repository = _get_remote()
437 args.append(repository)
439 args.extend(refspecs)
440 with _tempfile.TemporaryDirectory(prefix='nmbug-pull.') as workdir:
446 additional_env={'GIT_WORK_TREE': workdir},
451 def merge(reference='@{upstream}'):
453 Merge changes from 'reference' into HEAD and load the result into notmuch.
455 The default reference is '@{upstream}'.
458 with _tempfile.TemporaryDirectory(prefix='nmbug-merge.') as workdir:
461 ['merge', reference]]:
464 additional_env={'GIT_WORK_TREE': workdir},
471 A simple wrapper for 'git log'.
473 After running 'nmbug fetch', you can inspect the changes with
474 'nmbug log HEAD..@{upstream}'.
476 # we don't want output trapping here, because we want the pager.
477 args = ['log', '--name-status'] + list(args)
478 with _git(args=args, expect=(0, 1, -13)) as p:
482 def push(repository=None, refspecs=None):
483 "Push the local nmbug Git state to a remote repository."
484 if refspecs and not repository:
485 repository = _get_remote()
488 args.append(repository)
490 args.extend(refspecs)
491 _git(args=args, wait=True)
496 Show pending updates in notmuch or git repo.
498 Prints lines of the form
502 where n is a single character representing notmuch database status
506 Tag is present in notmuch database, but not committed to nmbug
507 (equivalently, tag has been deleted in nmbug repo, e.g. by a
508 pull, but not restored to notmuch database).
512 Tag is present in nmbug repo, but not restored to notmuch
513 database (equivalently, tag has been deleted in notmuch).
517 Message is unknown (missing from local notmuch database).
519 The second character (if present) represents a difference between
520 local and upstream branches. Typically 'nmbug fetch' needs to be
525 Tag is present in upstream, but not in the local Git branch.
529 Tag is present in local Git branch, but not upstream.
531 status = get_status()
532 # 'output' is a nested defaultdict for message status:
533 # * The outer dict is keyed by message id.
534 # * The inner dict is keyed by tag name.
535 # * The inner dict values are status strings (' a', 'Dd', ...).
536 output = _collections.defaultdict(
537 lambda : _collections.defaultdict(lambda : ' '))
538 for id, tags in status['added'].items():
540 output[id][tag] = 'A'
541 for id, tags in status['deleted'].items():
543 output[id][tag] = 'D'
544 for id, tags in status['missing'].items():
546 output[id][tag] = 'U'
548 for id, tag in _diff_refs(filter='A'):
549 output[id][tag] += 'a'
550 for id, tag in _diff_refs(filter='D'):
551 output[id][tag] += 'd'
552 for id, tag_status in sorted(output.items()):
553 for tag, status in sorted(tag_status.items()):
554 print('{status}\t{id}\t{tag}'.format(
555 status=status, id=id, tag=tag))
558 def _is_unmerged(ref='@{upstream}'):
560 (status, fetch_head, stderr) = _git(
561 args=['rev-parse', ref],
562 stdout=_subprocess.PIPE, stderr=_subprocess.PIPE, wait=True)
563 except SubprocessError as e:
564 if 'No upstream configured' in e.stderr:
567 (status, base, stderr) = _git(
568 args=['merge-base', 'HEAD', ref],
569 stdout=_subprocess.PIPE, wait=True)
570 return base != fetch_head
578 index = _index_tags()
579 maybe_deleted = _diff_index(index=index, filter='D')
580 for id, tags in maybe_deleted.items():
581 (_, stdout, stderr) = _spawn(
582 args=['notmuch', 'search', '--output=files', 'id:{0}'.format(id)],
583 stdout=_subprocess.PIPE,
586 status['deleted'][id] = tags
588 status['missing'][id] = tags
589 status['added'] = _diff_index(index=index, filter='A')
595 "Write notmuch tags to the nmbug.index."
596 path = _os.path.join(NMBGIT, 'nmbug.index')
597 query = ' '.join('tag:"{tag}"'.format(tag=tag) for tag in get_tags())
598 prefix = '+{0}'.format(_ENCODED_TAG_PREFIX)
600 args=['read-tree', '--empty'],
601 additional_env={'GIT_INDEX_FILE': path}, wait=True)
603 args=['notmuch', 'dump', '--format=batch-tag', '--', query],
604 stdout=_subprocess.PIPE) as notmuch:
606 args=['update-index', '--index-info'],
607 stdin=_subprocess.PIPE,
608 additional_env={'GIT_INDEX_FILE': path}) as git:
609 for line in notmuch.stdout:
610 (tags_string, id) = [_.strip() for _ in line.split(' -- id:')]
612 _unquote(tag[len(prefix):])
613 for tag in tags_string.split()
614 if tag.startswith(prefix)]
615 id = _xapian_unquote(string=id)
616 for line in _index_tags_for_message(
617 id=id, status='A', tags=tags):
618 git.stdin.write(line)
622 def _index_tags_for_message(id, status, tags):
624 Update the Git index to either create or delete an empty file.
626 Neither 'id' nor the tags in 'tags' should be encoded/escaped.
633 hash = '0000000000000000000000000000000000000000'
636 path = 'tags/{id}/{tag}'.format(
637 id=_hex_quote(string=id), tag=_hex_quote(string=tag))
638 yield '{mode} {hash}\t{path}\n'.format(mode=mode, hash=hash, path=path)
641 def _diff_index(index, filter):
643 Get an {id: {tag, ...}} dict for a given filter.
645 For example, use 'A' to find added tags, and 'D' to find deleted tags.
647 s = _collections.defaultdict(set)
650 'diff-index', '--cached', '--diff-filter', filter,
651 '--name-only', 'HEAD'],
652 additional_env={'GIT_INDEX_FILE': index},
653 stdout=_subprocess.PIPE) as p:
654 # Once we drop Python < 3.3, we can use 'yield from' here
655 for id, tag in _unpack_diff_lines(stream=p.stdout):
660 def _diff_refs(filter, a='HEAD', b='@{upstream}'):
662 args=['diff', '--diff-filter', filter, '--name-only', a, b],
663 stdout=_subprocess.PIPE) as p:
664 # Once we drop Python < 3.3, we can use 'yield from' here
665 for id, tag in _unpack_diff_lines(stream=p.stdout):
669 def _unpack_diff_lines(stream):
670 "Iterate through (id, tag) tuples in a diff stream."
672 match = _TAG_FILE_REGEX.match(line.strip())
675 'Invalid line in diff: {!r}'.format(line.strip()))
676 id = _unquote(match.group('id'))
677 tag = _unquote(match.group('tag'))
681 def _help(parser, command=None):
683 Show help for an nmbug command.
685 Because some folks prefer:
691 $ nmbug COMMAND --help
694 parser.parse_args([command, '--help'])
696 parser.parse_args(['--help'])
699 if __name__ == '__main__':
702 parser = argparse.ArgumentParser(
703 description=__doc__.strip(),
704 formatter_class=argparse.RawDescriptionHelpFormatter)
706 '-v', '--version', action='version',
707 version='%(prog)s {}'.format(__version__))
710 choices=['critical', 'error', 'warning', 'info', 'debug'],
711 help='Log verbosity. Defaults to {!r}.'.format(
712 _logging.getLevelName(_LOG.level).lower()))
714 help = _functools.partial(_help, parser=parser)
715 help.__doc__ = _help.__doc__
716 subparsers = parser.add_subparsers(
719 'For help on a particular command, run: '
720 "'%(prog)s ... <command> --help'."))
734 func = locals()[command]
735 doc = _textwrap.dedent(func.__doc__).strip().replace('%', '%%')
736 subparser = subparsers.add_parser(
738 help=doc.splitlines()[0],
740 formatter_class=argparse.RawDescriptionHelpFormatter)
741 subparser.set_defaults(func=func)
742 if command == 'archive':
743 subparser.add_argument(
744 'treeish', metavar='TREE-ISH', nargs='?', default='HEAD',
746 'The tree or commit to produce an archive for. Defaults '
748 subparser.add_argument(
749 'args', metavar='ARG', nargs='*',
751 "Argument passed through to 'git archive'. Set anything "
752 'before <tree-ish>, see git-archive(1) for details.'))
753 elif command == 'clone':
754 subparser.add_argument(
757 'The (possibly remote) repository to clone from. See the '
758 'URLS section of git-clone(1) for more information on '
759 'specifying repositories.'))
760 elif command == 'commit':
761 subparser.add_argument(
762 'message', metavar='MESSAGE', default='', nargs='?',
763 help='Text for the commit message.')
764 elif command == 'fetch':
765 subparser.add_argument(
766 'remote', metavar='REMOTE', nargs='?',
768 'Override the default configured in branch.<name>.remote '
769 'to fetch from a particular remote repository (e.g. '
771 elif command == 'help':
772 subparser.add_argument(
773 'command', metavar='COMMAND', nargs='?',
774 help='The command to show help for.')
775 elif command == 'log':
776 subparser.add_argument(
777 'args', metavar='ARG', nargs='*',
778 help="Additional argument passed through to 'git log'.")
779 elif command == 'merge':
780 subparser.add_argument(
781 'reference', metavar='REFERENCE', default='@{upstream}',
784 'Reference, usually other branch heads, to merge into '
785 "our branch. Defaults to '@{upstream}'."))
786 elif command == 'pull':
787 subparser.add_argument(
788 'repository', metavar='REPOSITORY', default=None, nargs='?',
790 'The "remote" repository that is the source of the pull. '
791 'This parameter can be either a URL (see the section GIT '
792 'URLS in git-pull(1)) or the name of a remote (see the '
793 'section REMOTES in git-pull(1)).'))
794 subparser.add_argument(
795 'refspecs', metavar='REFSPEC', default=None, nargs='*',
797 'Refspec (usually a branch name) to fetch and merge. See '
798 'the <refspec> entry in the OPTIONS section of '
799 'git-pull(1) for other possibilities.'))
800 elif command == 'push':
801 subparser.add_argument(
802 'repository', metavar='REPOSITORY', default=None, nargs='?',
804 'The "remote" repository that is the destination of the '
805 'push. This parameter can be either a URL (see the '
806 'section GIT URLS in git-push(1)) or the name of a remote '
807 '(see the section REMOTES in git-push(1)).'))
808 subparser.add_argument(
809 'refspecs', metavar='REFSPEC', default=None, nargs='*',
811 'Refspec (usually a branch name) to push. See '
812 'the <refspec> entry in the OPTIONS section of '
813 'git-push(1) for other possibilities.'))
815 args = parser.parse_args()
818 level = getattr(_logging, args.log_level.upper())
821 if not getattr(args, 'func', None):
825 if args.func == help:
826 arg_names = ['command']
828 (arg_names, varargs, varkw) = _inspect.getargs(args.func.__code__)
829 kwargs = {key: getattr(args, key) for key in arg_names if key in args}
832 except SubprocessError as e:
833 if _LOG.level == _logging.DEBUG:
834 raise # don't mask the traceback