3 # Copyright (c) 2011-2014 David Bremner <david@tethera.net>
4 # W. Trevor King <wking@tremily.us>
6 # This program is free software: you can redistribute it and/or modify
7 # it under the terms of the GNU General Public License as published by
8 # the Free Software Foundation, either version 3 of the License, or
9 # (at your option) any later version.
11 # This program is distributed in the hope that it will be useful,
12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 # GNU General Public License for more details.
16 # You should have received a copy of the GNU General Public License
17 # along with this program. If not, see http://www.gnu.org/licenses/ .
20 Manage notmuch tags with Git
22 Environment variables:
24 * NMBGIT specifies the location of the git repository used by nmbug.
25 If not specified $HOME/.nmbug is used.
26 * NMBPREFIX specifies the prefix in the notmuch database for tags of
27 interest to nmbug. If not specified 'notmuch::' is used.
30 from __future__ import print_function
31 from __future__ import unicode_literals
33 import codecs as _codecs
34 import collections as _collections
35 import functools as _functools
36 import inspect as _inspect
37 import locale as _locale
38 import logging as _logging
41 import shutil as _shutil
42 import subprocess as _subprocess
44 import tempfile as _tempfile
45 import textwrap as _textwrap
47 from urllib.parse import quote as _quote
48 from urllib.parse import unquote as _unquote
49 except ImportError: # Python 2
50 from urllib import quote as _quote
51 from urllib import unquote as _unquote
56 _LOG = _logging.getLogger('nmbug')
57 _LOG.setLevel(_logging.ERROR)
58 _LOG.addHandler(_logging.StreamHandler())
60 NMBGIT = _os.path.expanduser(
61 _os.getenv('NMBGIT', _os.path.join('~', '.nmbug')))
62 _NMBGIT = _os.path.join(NMBGIT, '.git')
63 if _os.path.isdir(_NMBGIT):
66 TAG_PREFIX = _os.getenv('NMBPREFIX', 'notmuch::')
67 _HEX_ESCAPE_REGEX = _re.compile('%[0-9A-F]{2}')
68 _TAG_FILE_REGEX = _re.compile('tags/(?P<id>[^/]*)/(?P<tag>[^/]*)')
70 # magic hash for Git (git hash-object -t blob /dev/null)
71 _EMPTYBLOB = 'e69de29bb2d1d6434b8b29ae775ad8c2e48c5391'
75 getattr(_tempfile, 'TemporaryDirectory')
76 except AttributeError: # Python < 3.2
77 class _TemporaryDirectory(object):
79 Fallback context manager for Python < 3.2
81 See PEP 343 for details on context managers [1].
83 [1]: http://legacy.python.org/dev/peps/pep-0343/
85 def __init__(self, **kwargs):
86 self.name = _tempfile.mkdtemp(**kwargs)
91 def __exit__(self, type, value, traceback):
92 _shutil.rmtree(self.name)
95 _tempfile.TemporaryDirectory = _TemporaryDirectory
98 def _hex_quote(string, safe='+@=:,'):
100 quote('abc def') -> 'abc%20def'.
102 Wrap urllib.parse.quote with additional safe characters (in
103 addition to letters, digits, and '_.-') and lowercase hex digits
104 (e.g. '%3a' instead of '%3A').
106 uppercase_escapes = _quote(string, safe)
107 return _HEX_ESCAPE_REGEX.sub(
108 lambda match: match.group(0).lower(),
112 _ENCODED_TAG_PREFIX = _hex_quote(TAG_PREFIX, safe='+@=,') # quote ':'
115 def _xapian_quote(string):
117 Quote a string for Xapian's QueryParser.
119 Xapian uses double-quotes for quoting strings. You can escape
120 internal quotes by repeating them [1,2,3].
122 [1]: http://trac.xapian.org/ticket/128#comment:2
123 [2]: http://trac.xapian.org/ticket/128#comment:17
124 [3]: http://trac.xapian.org/changeset/13823/svn
126 return '"{0}"'.format(string.replace('"', '""'))
129 def _xapian_unquote(string):
131 Unquote a Xapian-quoted string.
133 if string.startswith('"') and string.endswith('"'):
134 return string[1:-1].replace('""', '"')
138 class SubprocessError(RuntimeError):
139 "A subprocess exited with a nonzero status"
140 def __init__(self, args, status, stdout=None, stderr=None):
144 msg = '{args} exited with {status}'.format(args=args, status=status)
146 msg = '{msg}: {stderr}'.format(msg=msg, stderr=stderr)
147 super(SubprocessError, self).__init__(msg)
150 class _SubprocessContextManager(object):
152 PEP 343 context manager for subprocesses.
154 'expect' holds a tuple of acceptable exit codes, otherwise we'll
155 raise a SubprocessError in __exit__.
157 def __init__(self, process, args, expect=(0,)):
158 self._process = process
160 self._expect = expect
165 def __exit__(self, type, value, traceback):
166 for name in ['stdin', 'stdout', 'stderr']:
167 stream = getattr(self._process, name)
170 setattr(self._process, name, None)
171 status = self._process.wait()
172 _LOG.debug('collect {args} with status {status}'.format(
173 args=self._args, status=status))
174 if status not in self._expect:
175 raise SubprocessError(args=self._args, status=status)
178 return self._process.wait()
181 def _spawn(args, input=None, additional_env=None, wait=False, stdin=None,
182 stdout=None, stderr=None, encoding=_locale.getpreferredencoding(),
183 expect=(0,), **kwargs):
184 """Spawn a subprocess, and optionally wait for it to finish.
186 This wrapper around subprocess.Popen has two modes, depending on
187 the truthiness of 'wait'. If 'wait' is true, we use p.communicate
188 internally to write 'input' to the subprocess's stdin and read
189 from it's stdout/stderr. If 'wait' is False, we return a
190 _SubprocessContextManager instance for fancier handling
191 (e.g. piping between processes).
193 For 'wait' calls when you want to write to the subprocess's stdin,
194 you only need to set 'input' to your content. When 'input' is not
195 None but 'stdin' is, we'll automatically set 'stdin' to PIPE
196 before calling Popen. This avoids having the subprocess
197 accidentally inherit the launching process's stdin.
199 _LOG.debug('spawn {args} (additional env. var.: {env})'.format(
200 args=args, env=additional_env))
201 if not stdin and input is not None:
202 stdin = _subprocess.PIPE
204 if not kwargs.get('env'):
205 kwargs['env'] = dict(_os.environ)
206 kwargs['env'].update(additional_env)
207 p = _subprocess.Popen(
208 args, stdin=stdin, stdout=stdout, stderr=stderr, **kwargs)
210 if hasattr(input, 'encode'):
211 input = input.encode(encoding)
212 (stdout, stderr) = p.communicate(input=input)
214 _LOG.debug('collect {args} with status {status}'.format(
215 args=args, status=status))
216 if stdout is not None:
217 stdout = stdout.decode(encoding)
218 if stderr is not None:
219 stderr = stderr.decode(encoding)
221 raise SubprocessError(
222 args=args, status=status, stdout=stdout, stderr=stderr)
223 return (status, stdout, stderr)
224 if p.stdin and not stdin:
228 p.stdin = _codecs.getwriter(encoding=encoding)(stream=p.stdin)
229 stream_reader = _codecs.getreader(encoding=encoding)
231 p.stdout = stream_reader(stream=p.stdout)
233 p.stderr = stream_reader(stream=p.stderr)
234 return _SubprocessContextManager(args=args, process=p, expect=expect)
237 def _git(args, **kwargs):
238 args = ['git', '--git-dir', NMBGIT] + list(args)
239 return _spawn(args=args, **kwargs)
242 def _get_current_branch():
243 """Get the name of the current branch.
245 Return 'None' if we're not on a branch.
248 (status, branch, stderr) = _git(
249 args=['symbolic-ref', '--short', 'HEAD'],
250 stdout=_subprocess.PIPE, stderr=_subprocess.PIPE, wait=True)
251 except SubprocessError as e:
252 if 'not a symbolic ref' in e:
255 return branch.strip()
259 "Get the default remote for the current branch."
260 local_branch = _get_current_branch()
261 (status, remote, stderr) = _git(
262 args=['config', 'branch.{0}.remote'.format(local_branch)],
263 stdout=_subprocess.PIPE, wait=True)
264 return remote.strip()
267 def get_tags(prefix=None):
268 "Get a list of tags with a given prefix."
271 (status, stdout, stderr) = _spawn(
272 args=['notmuch', 'search', '--output=tags', '*'],
273 stdout=_subprocess.PIPE, wait=True)
274 return [tag for tag in stdout.splitlines() if tag.startswith(prefix)]
277 def archive(treeish='HEAD', args=()):
279 Dump a tar archive of the current nmbug tag set.
283 Each tag $tag for message with Message-Id $id is written to
286 tags/encode($id)/encode($tag)
288 The encoding preserves alphanumerics, and the characters
289 "+-_@=.:," (not the quotes). All other octets are replaced with
290 '%' followed by a two digit hex number.
292 _git(args=['archive', treeish] + list(args), wait=True)
295 def clone(repository):
297 Create a local nmbug repository from a remote source.
299 This wraps 'git clone', adding some options to avoid creating a
300 working tree while preserving remote-tracking branches and
303 with _tempfile.TemporaryDirectory(prefix='nmbug-clone.') as workdir:
306 'git', 'clone', '--no-checkout', '--separate-git-dir', NMBGIT,
307 repository, workdir],
309 _git(args=['config', '--unset', 'core.worktree'], wait=True)
310 _git(args=['config', 'core.bare', 'true'], wait=True)
311 _git(args=['branch', 'config', 'origin/config'], wait=True)
314 def _is_committed(status):
315 return len(status['added']) + len(status['deleted']) == 0
318 def commit(treeish='HEAD', message=None):
320 Commit prefix-matching tags from the notmuch database to Git.
322 status = get_status()
324 if _is_committed(status=status):
325 _LOG.warning('Nothing to commit')
328 _git(args=['read-tree', '--empty'], wait=True)
329 _git(args=['read-tree', treeish], wait=True)
331 _update_index(status=status)
334 stdout=_subprocess.PIPE,
336 (_, parent, _) = _git(
337 args=['rev-parse', treeish],
338 stdout=_subprocess.PIPE,
340 (_, commit, _) = _git(
341 args=['commit-tree', tree.strip(), '-p', parent.strip()],
343 stdout=_subprocess.PIPE,
346 args=['update-ref', treeish, commit.strip()],
347 stdout=_subprocess.PIPE,
349 except Exception as e:
350 _git(args=['read-tree', '--empty'], wait=True)
351 _git(args=['read-tree', treeish], wait=True)
354 def _update_index(status):
356 args=['update-index', '--index-info'],
357 stdin=_subprocess.PIPE) as p:
358 for id, tags in status['deleted'].items():
359 for line in _index_tags_for_message(id=id, status='D', tags=tags):
361 for id, tags in status['added'].items():
362 for line in _index_tags_for_message(id=id, status='A', tags=tags):
366 def fetch(remote=None):
368 Fetch changes from the remote repository.
370 See 'merge' to bring those changes into notmuch.
375 _git(args=args, wait=True)
380 Update the notmuch database from Git.
382 This is mainly useful to discard your changes in notmuch relative
385 status = get_status()
387 args=['notmuch', 'tag', '--batch'], stdin=_subprocess.PIPE) as p:
388 for id, tags in status['added'].items():
389 p.stdin.write(_batch_line(action='-', id=id, tags=tags))
390 for id, tags in status['deleted'].items():
391 p.stdin.write(_batch_line(action='+', id=id, tags=tags))
394 def _batch_line(action, id, tags):
396 'notmuch tag --batch' line for adding/removing tags.
398 Set 'action' to '-' to remove a tag or '+' to add the tags to a
401 tag_string = ' '.join(
402 '{action}{prefix}{tag}'.format(
403 action=action, prefix=_ENCODED_TAG_PREFIX, tag=_hex_quote(tag))
405 line = '{tags} -- id:{id}\n'.format(
406 tags=tag_string, id=_xapian_quote(string=id))
410 def _insist_committed():
411 "Die if the the notmuch tags don't match the current HEAD."
412 status = get_status()
413 if not _is_committed(status=status):
414 _LOG.error('\n'.join([
415 'Uncommitted changes to {prefix}* tags in notmuch',
417 "For a summary of changes, run 'nmbug status'",
418 "To save your changes, run 'nmbug commit' before merging/pull",
419 "To discard your changes, run 'nmbug checkout'",
420 ]).format(prefix=TAG_PREFIX))
424 def pull(repository=None, refspecs=None):
426 Pull (merge) remote repository changes to notmuch.
428 'pull' is equivalent to 'fetch' followed by 'merge'. We use the
429 Git-configured repository for your current branch
430 (branch.<name>.repository, likely 'origin', and
431 branch.<name>.merge, likely 'master').
434 if refspecs and not repository:
435 repository = _get_remote()
438 args.append(repository)
440 args.extend(refspecs)
441 with _tempfile.TemporaryDirectory(prefix='nmbug-pull.') as workdir:
447 additional_env={'GIT_WORK_TREE': workdir},
452 def merge(reference='@{upstream}'):
454 Merge changes from 'reference' into HEAD and load the result into notmuch.
456 The default reference is '@{upstream}'.
459 with _tempfile.TemporaryDirectory(prefix='nmbug-merge.') as workdir:
462 ['merge', reference]]:
465 additional_env={'GIT_WORK_TREE': workdir},
472 A simple wrapper for 'git log'.
474 After running 'nmbug fetch', you can inspect the changes with
475 'nmbug log HEAD..@{upstream}'.
477 # we don't want output trapping here, because we want the pager.
478 args = ['log', '--name-status'] + list(args)
479 with _git(args=args, expect=(0, 1, -13)) as p:
483 def push(repository=None, refspecs=None):
484 "Push the local nmbug Git state to a remote repository."
485 if refspecs and not repository:
486 repository = _get_remote()
489 args.append(repository)
491 args.extend(refspecs)
492 _git(args=args, wait=True)
497 Show pending updates in notmuch or git repo.
499 Prints lines of the form
503 where n is a single character representing notmuch database status
507 Tag is present in notmuch database, but not committed to nmbug
508 (equivalently, tag has been deleted in nmbug repo, e.g. by a
509 pull, but not restored to notmuch database).
513 Tag is present in nmbug repo, but not restored to notmuch
514 database (equivalently, tag has been deleted in notmuch).
518 Message is unknown (missing from local notmuch database).
520 The second character (if present) represents a difference between
521 local and upstream branches. Typically 'nmbug fetch' needs to be
526 Tag is present in upstream, but not in the local Git branch.
530 Tag is present in local Git branch, but not upstream.
532 status = get_status()
533 # 'output' is a nested defaultdict for message status:
534 # * The outer dict is keyed by message id.
535 # * The inner dict is keyed by tag name.
536 # * The inner dict values are status strings (' a', 'Dd', ...).
537 output = _collections.defaultdict(
538 lambda : _collections.defaultdict(lambda : ' '))
539 for id, tags in status['added'].items():
541 output[id][tag] = 'A'
542 for id, tags in status['deleted'].items():
544 output[id][tag] = 'D'
545 for id, tags in status['missing'].items():
547 output[id][tag] = 'U'
549 for id, tag in _diff_refs(filter='A'):
550 output[id][tag] += 'a'
551 for id, tag in _diff_refs(filter='D'):
552 output[id][tag] += 'd'
553 for id, tag_status in sorted(output.items()):
554 for tag, status in sorted(tag_status.items()):
555 print('{status}\t{id}\t{tag}'.format(
556 status=status, id=id, tag=tag))
559 def _is_unmerged(ref='@{upstream}'):
561 (status, fetch_head, stderr) = _git(
562 args=['rev-parse', ref],
563 stdout=_subprocess.PIPE, stderr=_subprocess.PIPE, wait=True)
564 except SubprocessError as e:
565 if 'No upstream configured' in e.stderr:
568 (status, base, stderr) = _git(
569 args=['merge-base', 'HEAD', ref],
570 stdout=_subprocess.PIPE, wait=True)
571 return base != fetch_head
579 index = _index_tags()
580 maybe_deleted = _diff_index(index=index, filter='D')
581 for id, tags in maybe_deleted.items():
582 (_, stdout, stderr) = _spawn(
583 args=['notmuch', 'search', '--output=files', 'id:{0}'.format(id)],
584 stdout=_subprocess.PIPE,
587 status['deleted'][id] = tags
589 status['missing'][id] = tags
590 status['added'] = _diff_index(index=index, filter='A')
596 "Write notmuch tags to the nmbug.index."
597 path = _os.path.join(NMBGIT, 'nmbug.index')
598 query = ' '.join('tag:"{tag}"'.format(tag=tag) for tag in get_tags())
599 prefix = '+{0}'.format(_ENCODED_TAG_PREFIX)
601 args=['read-tree', '--empty'],
602 additional_env={'GIT_INDEX_FILE': path}, wait=True)
604 args=['notmuch', 'dump', '--format=batch-tag', '--', query],
605 stdout=_subprocess.PIPE) as notmuch:
607 args=['update-index', '--index-info'],
608 stdin=_subprocess.PIPE,
609 additional_env={'GIT_INDEX_FILE': path}) as git:
610 for line in notmuch.stdout:
611 (tags_string, id) = [_.strip() for _ in line.split(' -- id:')]
613 _unquote(tag[len(prefix):])
614 for tag in tags_string.split()
615 if tag.startswith(prefix)]
616 id = _xapian_unquote(string=id)
617 for line in _index_tags_for_message(
618 id=id, status='A', tags=tags):
619 git.stdin.write(line)
623 def _index_tags_for_message(id, status, tags):
625 Update the Git index to either create or delete an empty file.
627 Neither 'id' nor the tags in 'tags' should be encoded/escaped.
634 hash = '0000000000000000000000000000000000000000'
637 path = 'tags/{id}/{tag}'.format(
638 id=_hex_quote(string=id), tag=_hex_quote(string=tag))
639 yield '{mode} {hash}\t{path}\n'.format(mode=mode, hash=hash, path=path)
642 def _diff_index(index, filter):
644 Get an {id: {tag, ...}} dict for a given filter.
646 For example, use 'A' to find added tags, and 'D' to find deleted tags.
648 s = _collections.defaultdict(set)
651 'diff-index', '--cached', '--diff-filter', filter,
652 '--name-only', 'HEAD'],
653 additional_env={'GIT_INDEX_FILE': index},
654 stdout=_subprocess.PIPE) as p:
655 # Once we drop Python < 3.3, we can use 'yield from' here
656 for id, tag in _unpack_diff_lines(stream=p.stdout):
661 def _diff_refs(filter, a='HEAD', b='@{upstream}'):
663 args=['diff', '--diff-filter', filter, '--name-only', a, b],
664 stdout=_subprocess.PIPE) as p:
665 # Once we drop Python < 3.3, we can use 'yield from' here
666 for id, tag in _unpack_diff_lines(stream=p.stdout):
670 def _unpack_diff_lines(stream):
671 "Iterate through (id, tag) tuples in a diff stream."
673 match = _TAG_FILE_REGEX.match(line.strip())
676 'Invalid line in diff: {!r}'.format(line.strip()))
677 id = _unquote(match.group('id'))
678 tag = _unquote(match.group('tag'))
682 def _help(parser, command=None):
684 Show help for an nmbug command.
686 Because some folks prefer:
692 $ nmbug COMMAND --help
695 parser.parse_args([command, '--help'])
697 parser.parse_args(['--help'])
700 if __name__ == '__main__':
703 parser = argparse.ArgumentParser(
704 description=__doc__.strip(),
705 formatter_class=argparse.RawDescriptionHelpFormatter)
707 '-v', '--version', action='version',
708 version='%(prog)s {}'.format(__version__))
711 choices=['critical', 'error', 'warning', 'info', 'debug'],
712 help='Log verbosity. Defaults to {!r}.'.format(
713 _logging.getLevelName(_LOG.level).lower()))
715 help = _functools.partial(_help, parser=parser)
716 help.__doc__ = _help.__doc__
717 subparsers = parser.add_subparsers(
720 'For help on a particular command, run: '
721 "'%(prog)s ... <command> --help'."))
735 func = locals()[command]
736 doc = _textwrap.dedent(func.__doc__).strip().replace('%', '%%')
737 subparser = subparsers.add_parser(
739 help=doc.splitlines()[0],
741 formatter_class=argparse.RawDescriptionHelpFormatter)
742 subparser.set_defaults(func=func)
743 if command == 'archive':
744 subparser.add_argument(
745 'treeish', metavar='TREE-ISH', nargs='?', default='HEAD',
747 'The tree or commit to produce an archive for. Defaults '
749 subparser.add_argument(
750 'args', metavar='ARG', nargs='*',
752 "Argument passed through to 'git archive'. Set anything "
753 'before <tree-ish>, see git-archive(1) for details.'))
754 elif command == 'clone':
755 subparser.add_argument(
758 'The (possibly remote) repository to clone from. See the '
759 'URLS section of git-clone(1) for more information on '
760 'specifying repositories.'))
761 elif command == 'commit':
762 subparser.add_argument(
763 'message', metavar='MESSAGE', default='', nargs='?',
764 help='Text for the commit message.')
765 elif command == 'fetch':
766 subparser.add_argument(
767 'remote', metavar='REMOTE', nargs='?',
769 'Override the default configured in branch.<name>.remote '
770 'to fetch from a particular remote repository (e.g. '
772 elif command == 'help':
773 subparser.add_argument(
774 'command', metavar='COMMAND', nargs='?',
775 help='The command to show help for.')
776 elif command == 'log':
777 subparser.add_argument(
778 'args', metavar='ARG', nargs='*',
779 help="Additional argument passed through to 'git log'.")
780 elif command == 'merge':
781 subparser.add_argument(
782 'reference', metavar='REFERENCE', default='@{upstream}',
785 'Reference, usually other branch heads, to merge into '
786 "our branch. Defaults to '@{upstream}'."))
787 elif command == 'pull':
788 subparser.add_argument(
789 'repository', metavar='REPOSITORY', default=None, nargs='?',
791 'The "remote" repository that is the source of the pull. '
792 'This parameter can be either a URL (see the section GIT '
793 'URLS in git-pull(1)) or the name of a remote (see the '
794 'section REMOTES in git-pull(1)).'))
795 subparser.add_argument(
796 'refspecs', metavar='REFSPEC', default=None, nargs='*',
798 'Refspec (usually a branch name) to fetch and merge. See '
799 'the <refspec> entry in the OPTIONS section of '
800 'git-pull(1) for other possibilities.'))
801 elif command == 'push':
802 subparser.add_argument(
803 'repository', metavar='REPOSITORY', default=None, nargs='?',
805 'The "remote" repository that is the destination of the '
806 'push. This parameter can be either a URL (see the '
807 'section GIT URLS in git-push(1)) or the name of a remote '
808 '(see the section REMOTES in git-push(1)).'))
809 subparser.add_argument(
810 'refspecs', metavar='REFSPEC', default=None, nargs='*',
812 'Refspec (usually a branch name) to push. See '
813 'the <refspec> entry in the OPTIONS section of '
814 'git-push(1) for other possibilities.'))
816 args = parser.parse_args()
819 level = getattr(_logging, args.log_level.upper())
822 if not getattr(args, 'func', None):
826 if args.func == help:
827 arg_names = ['command']
829 (arg_names, varargs, varkw) = _inspect.getargs(args.func.__code__)
830 kwargs = {key: getattr(args, key) for key in arg_names if key in args}
833 except SubprocessError as e:
834 if _LOG.level == _logging.DEBUG:
835 raise # don't mask the traceback