3 # Copyright (c) 2011-2014 David Bremner <david@tethera.net>
4 # W. Trevor King <wking@tremily.us>
6 # This program is free software: you can redistribute it and/or modify
7 # it under the terms of the GNU General Public License as published by
8 # the Free Software Foundation, either version 3 of the License, or
9 # (at your option) any later version.
11 # This program is distributed in the hope that it will be useful,
12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 # GNU General Public License for more details.
16 # You should have received a copy of the GNU General Public License
17 # along with this program. If not, see https://www.gnu.org/licenses/ .
20 Manage notmuch tags with Git
22 Environment variables:
24 * NMBGIT specifies the location of the git repository used by nmbug.
25 If not specified $HOME/.nmbug is used.
26 * NMBPREFIX specifies the prefix in the notmuch database for tags of
27 interest to nmbug. If not specified 'notmuch::' is used.
30 from __future__ import print_function
31 from __future__ import unicode_literals
33 import codecs as _codecs
34 import collections as _collections
35 import functools as _functools
36 import inspect as _inspect
37 import locale as _locale
38 import logging as _logging
41 import shutil as _shutil
42 import subprocess as _subprocess
44 import tempfile as _tempfile
45 import textwrap as _textwrap
47 from urllib.parse import quote as _quote
48 from urllib.parse import unquote as _unquote
49 except ImportError: # Python 2
50 from urllib import quote as _quote
51 from urllib import unquote as _unquote
56 _LOG = _logging.getLogger('nmbug')
57 _LOG.setLevel(_logging.WARNING)
58 _LOG.addHandler(_logging.StreamHandler())
60 NMBGIT = _os.path.expanduser(
61 _os.getenv('NMBGIT', _os.path.join('~', '.nmbug')))
62 _NMBGIT = _os.path.join(NMBGIT, '.git')
63 if _os.path.isdir(_NMBGIT):
66 TAG_PREFIX = _os.getenv('NMBPREFIX', 'notmuch::')
67 _HEX_ESCAPE_REGEX = _re.compile('%[0-9A-F]{2}')
68 _TAG_DIRECTORY = 'tags/'
69 _TAG_FILE_REGEX = _re.compile(_TAG_DIRECTORY + '(?P<id>[^/]*)/(?P<tag>[^/]*)')
71 # magic hash for Git (git hash-object -t blob /dev/null)
72 _EMPTYBLOB = 'e69de29bb2d1d6434b8b29ae775ad8c2e48c5391'
76 getattr(_tempfile, 'TemporaryDirectory')
77 except AttributeError: # Python < 3.2
78 class _TemporaryDirectory(object):
80 Fallback context manager for Python < 3.2
82 See PEP 343 for details on context managers [1].
84 [1]: https://www.python.org/dev/peps/pep-0343/
86 def __init__(self, **kwargs):
87 self.name = _tempfile.mkdtemp(**kwargs)
92 def __exit__(self, type, value, traceback):
93 _shutil.rmtree(self.name)
96 _tempfile.TemporaryDirectory = _TemporaryDirectory
99 def _hex_quote(string, safe='+@=:,'):
101 quote('abc def') -> 'abc%20def'.
103 Wrap urllib.parse.quote with additional safe characters (in
104 addition to letters, digits, and '_.-') and lowercase hex digits
105 (e.g. '%3a' instead of '%3A').
107 uppercase_escapes = _quote(string, safe)
108 return _HEX_ESCAPE_REGEX.sub(
109 lambda match: match.group(0).lower(),
113 _ENCODED_TAG_PREFIX = _hex_quote(TAG_PREFIX, safe='+@=,') # quote ':'
116 def _xapian_quote(string):
118 Quote a string for Xapian's QueryParser.
120 Xapian uses double-quotes for quoting strings. You can escape
121 internal quotes by repeating them [1,2,3].
123 [1]: https://trac.xapian.org/ticket/128#comment:2
124 [2]: https://trac.xapian.org/ticket/128#comment:17
125 [3]: https://trac.xapian.org/changeset/13823/svn
127 return '"{0}"'.format(string.replace('"', '""'))
130 def _xapian_unquote(string):
132 Unquote a Xapian-quoted string.
134 if string.startswith('"') and string.endswith('"'):
135 return string[1:-1].replace('""', '"')
139 class SubprocessError(RuntimeError):
140 "A subprocess exited with a nonzero status"
141 def __init__(self, args, status, stdout=None, stderr=None):
145 msg = '{args} exited with {status}'.format(args=args, status=status)
147 msg = '{msg}: {stderr}'.format(msg=msg, stderr=stderr)
148 super(SubprocessError, self).__init__(msg)
151 class _SubprocessContextManager(object):
153 PEP 343 context manager for subprocesses.
155 'expect' holds a tuple of acceptable exit codes, otherwise we'll
156 raise a SubprocessError in __exit__.
158 def __init__(self, process, args, expect=(0,)):
159 self._process = process
161 self._expect = expect
166 def __exit__(self, type, value, traceback):
167 for name in ['stdin', 'stdout', 'stderr']:
168 stream = getattr(self._process, name)
171 setattr(self._process, name, None)
172 status = self._process.wait()
174 'collect {args} with status {status} (expected {expect})'.format(
175 args=self._args, status=status, expect=self._expect))
176 if status not in self._expect:
177 raise SubprocessError(args=self._args, status=status)
180 return self._process.wait()
183 def _spawn(args, input=None, additional_env=None, wait=False, stdin=None,
184 stdout=None, stderr=None, encoding=_locale.getpreferredencoding(),
185 expect=(0,), **kwargs):
186 """Spawn a subprocess, and optionally wait for it to finish.
188 This wrapper around subprocess.Popen has two modes, depending on
189 the truthiness of 'wait'. If 'wait' is true, we use p.communicate
190 internally to write 'input' to the subprocess's stdin and read
191 from it's stdout/stderr. If 'wait' is False, we return a
192 _SubprocessContextManager instance for fancier handling
193 (e.g. piping between processes).
195 For 'wait' calls when you want to write to the subprocess's stdin,
196 you only need to set 'input' to your content. When 'input' is not
197 None but 'stdin' is, we'll automatically set 'stdin' to PIPE
198 before calling Popen. This avoids having the subprocess
199 accidentally inherit the launching process's stdin.
201 _LOG.debug('spawn {args} (additional env. var.: {env})'.format(
202 args=args, env=additional_env))
203 if not stdin and input is not None:
204 stdin = _subprocess.PIPE
206 if not kwargs.get('env'):
207 kwargs['env'] = dict(_os.environ)
208 kwargs['env'].update(additional_env)
209 p = _subprocess.Popen(
210 args, stdin=stdin, stdout=stdout, stderr=stderr, **kwargs)
212 if hasattr(input, 'encode'):
213 input = input.encode(encoding)
214 (stdout, stderr) = p.communicate(input=input)
217 'collect {args} with status {status} (expected {expect})'.format(
218 args=args, status=status, expect=expect))
219 if stdout is not None:
220 stdout = stdout.decode(encoding)
221 if stderr is not None:
222 stderr = stderr.decode(encoding)
223 if status not in expect:
224 raise SubprocessError(
225 args=args, status=status, stdout=stdout, stderr=stderr)
226 return (status, stdout, stderr)
227 if p.stdin and not stdin:
231 p.stdin = _codecs.getwriter(encoding=encoding)(stream=p.stdin)
232 stream_reader = _codecs.getreader(encoding=encoding)
234 p.stdout = stream_reader(stream=p.stdout)
236 p.stderr = stream_reader(stream=p.stderr)
237 return _SubprocessContextManager(args=args, process=p, expect=expect)
240 def _git(args, **kwargs):
241 args = ['git', '--git-dir', NMBGIT] + list(args)
242 return _spawn(args=args, **kwargs)
245 def _get_current_branch():
246 """Get the name of the current branch.
248 Return 'None' if we're not on a branch.
251 (status, branch, stderr) = _git(
252 args=['symbolic-ref', '--short', 'HEAD'],
253 stdout=_subprocess.PIPE, stderr=_subprocess.PIPE, wait=True)
254 except SubprocessError as e:
255 if 'not a symbolic ref' in e:
258 return branch.strip()
262 "Get the default remote for the current branch."
263 local_branch = _get_current_branch()
264 (status, remote, stderr) = _git(
265 args=['config', 'branch.{0}.remote'.format(local_branch)],
266 stdout=_subprocess.PIPE, wait=True)
267 return remote.strip()
270 def get_tags(prefix=None):
271 "Get a list of tags with a given prefix."
274 (status, stdout, stderr) = _spawn(
275 args=['notmuch', 'search', '--output=tags', '*'],
276 stdout=_subprocess.PIPE, wait=True)
277 return [tag for tag in stdout.splitlines() if tag.startswith(prefix)]
280 def archive(treeish='HEAD', args=()):
282 Dump a tar archive of the current nmbug tag set.
286 Each tag $tag for message with Message-Id $id is written to
289 tags/encode($id)/encode($tag)
291 The encoding preserves alphanumerics, and the characters
292 "+-_@=.:," (not the quotes). All other octets are replaced with
293 '%' followed by a two digit hex number.
295 _git(args=['archive', treeish] + list(args), wait=True)
298 def clone(repository):
300 Create a local nmbug repository from a remote source.
302 This wraps 'git clone', adding some options to avoid creating a
303 working tree while preserving remote-tracking branches and
306 with _tempfile.TemporaryDirectory(prefix='nmbug-clone.') as workdir:
309 'git', 'clone', '--no-checkout', '--separate-git-dir', NMBGIT,
310 repository, workdir],
312 _git(args=['config', '--unset', 'core.worktree'], wait=True, expect=(0, 5))
313 _git(args=['config', 'core.bare', 'true'], wait=True)
314 _git(args=['branch', 'config', 'origin/config'], wait=True)
315 existing_tags = get_tags()
318 'Not checking out to avoid clobbering existing tags: {}'.format(
319 ', '.join(existing_tags)))
324 def _is_committed(status):
325 return len(status['added']) + len(status['deleted']) == 0
328 def commit(treeish='HEAD', message=None):
330 Commit prefix-matching tags from the notmuch database to Git.
332 status = get_status()
334 if _is_committed(status=status):
335 _LOG.warning('Nothing to commit')
338 _git(args=['read-tree', '--empty'], wait=True)
339 _git(args=['read-tree', treeish], wait=True)
341 _update_index(status=status)
344 stdout=_subprocess.PIPE,
346 (_, parent, _) = _git(
347 args=['rev-parse', treeish],
348 stdout=_subprocess.PIPE,
350 (_, commit, _) = _git(
351 args=['commit-tree', tree.strip(), '-p', parent.strip()],
353 stdout=_subprocess.PIPE,
356 args=['update-ref', treeish, commit.strip()],
357 stdout=_subprocess.PIPE,
359 except Exception as e:
360 _git(args=['read-tree', '--empty'], wait=True)
361 _git(args=['read-tree', treeish], wait=True)
364 def _update_index(status):
366 args=['update-index', '--index-info'],
367 stdin=_subprocess.PIPE) as p:
368 for id, tags in status['deleted'].items():
369 for line in _index_tags_for_message(id=id, status='D', tags=tags):
371 for id, tags in status['added'].items():
372 for line in _index_tags_for_message(id=id, status='A', tags=tags):
376 def fetch(remote=None):
378 Fetch changes from the remote repository.
380 See 'merge' to bring those changes into notmuch.
385 _git(args=args, wait=True)
390 Update the notmuch database from Git.
392 This is mainly useful to discard your changes in notmuch relative
395 status = get_status()
397 args=['notmuch', 'tag', '--batch'], stdin=_subprocess.PIPE) as p:
398 for id, tags in status['added'].items():
399 p.stdin.write(_batch_line(action='-', id=id, tags=tags))
400 for id, tags in status['deleted'].items():
401 p.stdin.write(_batch_line(action='+', id=id, tags=tags))
404 def _batch_line(action, id, tags):
406 'notmuch tag --batch' line for adding/removing tags.
408 Set 'action' to '-' to remove a tag or '+' to add the tags to a
411 tag_string = ' '.join(
412 '{action}{prefix}{tag}'.format(
413 action=action, prefix=_ENCODED_TAG_PREFIX, tag=_hex_quote(tag))
415 line = '{tags} -- id:{id}\n'.format(
416 tags=tag_string, id=_xapian_quote(string=id))
420 def _insist_committed():
421 "Die if the the notmuch tags don't match the current HEAD."
422 status = get_status()
423 if not _is_committed(status=status):
424 _LOG.error('\n'.join([
425 'Uncommitted changes to {prefix}* tags in notmuch',
427 "For a summary of changes, run 'nmbug status'",
428 "To save your changes, run 'nmbug commit' before merging/pull",
429 "To discard your changes, run 'nmbug checkout'",
430 ]).format(prefix=TAG_PREFIX))
434 def pull(repository=None, refspecs=None):
436 Pull (merge) remote repository changes to notmuch.
438 'pull' is equivalent to 'fetch' followed by 'merge'. We use the
439 Git-configured repository for your current branch
440 (branch.<name>.repository, likely 'origin', and
441 branch.<name>.merge, likely 'master').
444 if refspecs and not repository:
445 repository = _get_remote()
448 args.append(repository)
450 args.extend(refspecs)
451 with _tempfile.TemporaryDirectory(prefix='nmbug-pull.') as workdir:
457 additional_env={'GIT_WORK_TREE': workdir},
462 def merge(reference='@{upstream}'):
464 Merge changes from 'reference' into HEAD and load the result into notmuch.
466 The default reference is '@{upstream}'.
469 with _tempfile.TemporaryDirectory(prefix='nmbug-merge.') as workdir:
472 ['merge', reference]]:
475 additional_env={'GIT_WORK_TREE': workdir},
482 A simple wrapper for 'git log'.
484 After running 'nmbug fetch', you can inspect the changes with
485 'nmbug log HEAD..@{upstream}'.
487 # we don't want output trapping here, because we want the pager.
488 args = ['log', '--name-status', '--no-renames'] + list(args)
489 with _git(args=args, expect=(0, 1, -13)) as p:
493 def push(repository=None, refspecs=None):
494 "Push the local nmbug Git state to a remote repository."
495 if refspecs and not repository:
496 repository = _get_remote()
499 args.append(repository)
501 args.extend(refspecs)
502 _git(args=args, wait=True)
507 Show pending updates in notmuch or git repo.
509 Prints lines of the form
513 where n is a single character representing notmuch database status
517 Tag is present in notmuch database, but not committed to nmbug
518 (equivalently, tag has been deleted in nmbug repo, e.g. by a
519 pull, but not restored to notmuch database).
523 Tag is present in nmbug repo, but not restored to notmuch
524 database (equivalently, tag has been deleted in notmuch).
528 Message is unknown (missing from local notmuch database).
530 The second character (if present) represents a difference between
531 local and upstream branches. Typically 'nmbug fetch' needs to be
536 Tag is present in upstream, but not in the local Git branch.
540 Tag is present in local Git branch, but not upstream.
542 status = get_status()
543 # 'output' is a nested defaultdict for message status:
544 # * The outer dict is keyed by message id.
545 # * The inner dict is keyed by tag name.
546 # * The inner dict values are status strings (' a', 'Dd', ...).
547 output = _collections.defaultdict(
548 lambda : _collections.defaultdict(lambda : ' '))
549 for id, tags in status['added'].items():
551 output[id][tag] = 'A'
552 for id, tags in status['deleted'].items():
554 output[id][tag] = 'D'
555 for id, tags in status['missing'].items():
557 output[id][tag] = 'U'
559 for id, tag in _diff_refs(filter='A'):
560 output[id][tag] += 'a'
561 for id, tag in _diff_refs(filter='D'):
562 output[id][tag] += 'd'
563 for id, tag_status in sorted(output.items()):
564 for tag, status in sorted(tag_status.items()):
565 print('{status}\t{id}\t{tag}'.format(
566 status=status, id=id, tag=tag))
569 def _is_unmerged(ref='@{upstream}'):
571 (status, fetch_head, stderr) = _git(
572 args=['rev-parse', ref],
573 stdout=_subprocess.PIPE, stderr=_subprocess.PIPE, wait=True)
574 except SubprocessError as e:
575 if 'No upstream configured' in e.stderr:
578 (status, base, stderr) = _git(
579 args=['merge-base', 'HEAD', ref],
580 stdout=_subprocess.PIPE, wait=True)
581 return base != fetch_head
589 index = _index_tags()
590 maybe_deleted = _diff_index(index=index, filter='D')
591 for id, tags in maybe_deleted.items():
592 (_, stdout, stderr) = _spawn(
593 args=['notmuch', 'search', '--output=files', 'id:{0}'.format(id)],
594 stdout=_subprocess.PIPE,
597 status['deleted'][id] = tags
599 status['missing'][id] = tags
600 status['added'] = _diff_index(index=index, filter='A')
606 "Write notmuch tags to the nmbug.index."
607 path = _os.path.join(NMBGIT, 'nmbug.index')
608 query = ' '.join('tag:"{tag}"'.format(tag=tag) for tag in get_tags())
609 prefix = '+{0}'.format(_ENCODED_TAG_PREFIX)
611 args=['read-tree', '--empty'],
612 additional_env={'GIT_INDEX_FILE': path}, wait=True)
614 args=['notmuch', 'dump', '--format=batch-tag', '--', query],
615 stdout=_subprocess.PIPE) as notmuch:
617 args=['update-index', '--index-info'],
618 stdin=_subprocess.PIPE,
619 additional_env={'GIT_INDEX_FILE': path}) as git:
620 for line in notmuch.stdout:
621 if line.strip().startswith('#'):
623 (tags_string, id) = [_.strip() for _ in line.split(' -- id:')]
625 _unquote(tag[len(prefix):])
626 for tag in tags_string.split()
627 if tag.startswith(prefix)]
628 id = _xapian_unquote(string=id)
629 for line in _index_tags_for_message(
630 id=id, status='A', tags=tags):
631 git.stdin.write(line)
635 def _index_tags_for_message(id, status, tags):
637 Update the Git index to either create or delete an empty file.
639 Neither 'id' nor the tags in 'tags' should be encoded/escaped.
646 hash = '0000000000000000000000000000000000000000'
649 path = 'tags/{id}/{tag}'.format(
650 id=_hex_quote(string=id), tag=_hex_quote(string=tag))
651 yield '{mode} {hash}\t{path}\n'.format(mode=mode, hash=hash, path=path)
654 def _diff_index(index, filter):
656 Get an {id: {tag, ...}} dict for a given filter.
658 For example, use 'A' to find added tags, and 'D' to find deleted tags.
660 s = _collections.defaultdict(set)
663 'diff-index', '--cached', '--diff-filter', filter,
664 '--name-only', 'HEAD'],
665 additional_env={'GIT_INDEX_FILE': index},
666 stdout=_subprocess.PIPE) as p:
667 # Once we drop Python < 3.3, we can use 'yield from' here
668 for id, tag in _unpack_diff_lines(stream=p.stdout):
673 def _diff_refs(filter, a='HEAD', b='@{upstream}'):
675 args=['diff', '--diff-filter', filter, '--name-only', a, b],
676 stdout=_subprocess.PIPE) as p:
677 # Once we drop Python < 3.3, we can use 'yield from' here
678 for id, tag in _unpack_diff_lines(stream=p.stdout):
682 def _unpack_diff_lines(stream):
683 "Iterate through (id, tag) tuples in a diff stream."
685 match = _TAG_FILE_REGEX.match(line.strip())
687 message = 'non-tag line in diff: {!r}'.format(line.strip())
688 if line.startswith(_TAG_DIRECTORY):
689 raise ValueError(message)
692 id = _unquote(match.group('id'))
693 tag = _unquote(match.group('tag'))
697 def _help(parser, command=None):
699 Show help for an nmbug command.
701 Because some folks prefer:
707 $ nmbug COMMAND --help
710 parser.parse_args([command, '--help'])
712 parser.parse_args(['--help'])
715 if __name__ == '__main__':
718 parser = argparse.ArgumentParser(
719 description=__doc__.strip(),
720 formatter_class=argparse.RawDescriptionHelpFormatter)
722 '-v', '--version', action='version',
723 version='%(prog)s {}'.format(__version__))
726 choices=['critical', 'error', 'warning', 'info', 'debug'],
727 help='Log verbosity. Defaults to {!r}.'.format(
728 _logging.getLevelName(_LOG.level).lower()))
730 help = _functools.partial(_help, parser=parser)
731 help.__doc__ = _help.__doc__
732 subparsers = parser.add_subparsers(
735 'For help on a particular command, run: '
736 "'%(prog)s ... <command> --help'."))
750 func = locals()[command]
751 doc = _textwrap.dedent(func.__doc__).strip().replace('%', '%%')
752 subparser = subparsers.add_parser(
754 help=doc.splitlines()[0],
756 formatter_class=argparse.RawDescriptionHelpFormatter)
757 subparser.set_defaults(func=func)
758 if command == 'archive':
759 subparser.add_argument(
760 'treeish', metavar='TREE-ISH', nargs='?', default='HEAD',
762 'The tree or commit to produce an archive for. Defaults '
764 subparser.add_argument(
765 'args', metavar='ARG', nargs='*',
767 "Argument passed through to 'git archive'. Set anything "
768 'before <tree-ish>, see git-archive(1) for details.'))
769 elif command == 'clone':
770 subparser.add_argument(
773 'The (possibly remote) repository to clone from. See the '
774 'URLS section of git-clone(1) for more information on '
775 'specifying repositories.'))
776 elif command == 'commit':
777 subparser.add_argument(
778 'message', metavar='MESSAGE', default='', nargs='?',
779 help='Text for the commit message.')
780 elif command == 'fetch':
781 subparser.add_argument(
782 'remote', metavar='REMOTE', nargs='?',
784 'Override the default configured in branch.<name>.remote '
785 'to fetch from a particular remote repository (e.g. '
787 elif command == 'help':
788 subparser.add_argument(
789 'command', metavar='COMMAND', nargs='?',
790 help='The command to show help for.')
791 elif command == 'log':
792 subparser.add_argument(
793 'args', metavar='ARG', nargs='*',
794 help="Additional argument passed through to 'git log'.")
795 elif command == 'merge':
796 subparser.add_argument(
797 'reference', metavar='REFERENCE', default='@{upstream}',
800 'Reference, usually other branch heads, to merge into '
801 "our branch. Defaults to '@{upstream}'."))
802 elif command == 'pull':
803 subparser.add_argument(
804 'repository', metavar='REPOSITORY', default=None, nargs='?',
806 'The "remote" repository that is the source of the pull. '
807 'This parameter can be either a URL (see the section GIT '
808 'URLS in git-pull(1)) or the name of a remote (see the '
809 'section REMOTES in git-pull(1)).'))
810 subparser.add_argument(
811 'refspecs', metavar='REFSPEC', default=None, nargs='*',
813 'Refspec (usually a branch name) to fetch and merge. See '
814 'the <refspec> entry in the OPTIONS section of '
815 'git-pull(1) for other possibilities.'))
816 elif command == 'push':
817 subparser.add_argument(
818 'repository', metavar='REPOSITORY', default=None, nargs='?',
820 'The "remote" repository that is the destination of the '
821 'push. This parameter can be either a URL (see the '
822 'section GIT URLS in git-push(1)) or the name of a remote '
823 '(see the section REMOTES in git-push(1)).'))
824 subparser.add_argument(
825 'refspecs', metavar='REFSPEC', default=None, nargs='*',
827 'Refspec (usually a branch name) to push. See '
828 'the <refspec> entry in the OPTIONS section of '
829 'git-push(1) for other possibilities.'))
831 args = parser.parse_args()
834 level = getattr(_logging, args.log_level.upper())
837 if not getattr(args, 'func', None):
841 if args.func == help:
842 arg_names = ['command']
844 (arg_names, varargs, varkw) = _inspect.getargs(args.func.__code__)
845 kwargs = {key: getattr(args, key) for key in arg_names if key in args}
848 except SubprocessError as e:
849 if _LOG.level == _logging.DEBUG:
850 raise # don't mask the traceback