X-Git-Url: https://git.cworth.org/git?a=blobdiff_plain;f=notmuch-git.py;h=aebff76474102311ad8f2515848e40a365c3bb38;hb=6219e7380ae34cc0c8142f4174bee3cde9bf9662;hp=e419cdf77e4abc340a81cb2b4ede4bdd7bb333f9;hpb=66ccf420c20bedec8dd78dc81ed1166761b7a467;p=notmuch diff --git a/notmuch-git.py b/notmuch-git.py index e419cdf7..aebff764 100644 --- a/notmuch-git.py +++ b/notmuch-git.py @@ -40,16 +40,18 @@ from urllib.parse import quote as _quote from urllib.parse import unquote as _unquote import json as _json -_LOG = _logging.getLogger('nmbug') +_LOG = _logging.getLogger('notmuch-git') _LOG.setLevel(_logging.WARNING) _LOG.addHandler(_logging.StreamHandler()) NOTMUCH_GIT_DIR = None TAG_PREFIX = None +FORMAT_VERSION = 0 _HEX_ESCAPE_REGEX = _re.compile('%[0-9A-F]{2}') _TAG_DIRECTORY = 'tags/' -_TAG_FILE_REGEX = _re.compile(_TAG_DIRECTORY + '(?P[^/]*)/(?P[^/]*)') +_TAG_FILE_REGEX = ( _re.compile(_TAG_DIRECTORY + '(?P[^/]*)/(?P[^/]*)'), + _re.compile(_TAG_DIRECTORY + '([0-9a-f]{2}/){2}(?P[^/]*)/(?P[^/]*)')) # magic hash for Git (git hash-object -t blob /dev/null) _EMPTYBLOB = 'e69de29bb2d1d6434b8b29ae775ad8c2e48c5391' @@ -239,6 +241,16 @@ def _tag_query(prefix=None): prefix = TAG_PREFIX return '(tag (starts-with "{:s}"))'.format(prefix.replace('"','\\\"')) +def count_messages(prefix=None): + "count messages with a given prefix." + (status, stdout, stderr) = _spawn( + args=['notmuch', 'count', '--query=sexp', _tag_query(prefix)], + stdout=_subprocess.PIPE, wait=True) + if status != 0: + _LOG.error("failed to run notmuch config") + sys.exit(1) + return int(stdout.rstrip()) + def get_tags(prefix=None): "Get a list of tags with a given prefix." (status, stdout, stderr) = _spawn( @@ -248,14 +260,14 @@ def get_tags(prefix=None): def archive(treeish='HEAD', args=()): """ - Dump a tar archive of the current nmbug tag set. + Dump a tar archive of the current notmuch-git tag set. Using 'git archive'. Each tag $tag for message with Message-Id $id is written to an empty file - tags/encode($id)/encode($tag) + tags/hash1(id)/hash2(id)/encode($id)/encode($tag) The encoding preserves alphanumerics, and the characters "+-_@=.:," (not the quotes). All other octets are replaced with @@ -266,13 +278,13 @@ def archive(treeish='HEAD', args=()): def clone(repository): """ - Create a local nmbug repository from a remote source. + Create a local notmuch-git repository from a remote source. This wraps 'git clone', adding some options to avoid creating a working tree while preserving remote-tracking branches and upstreams. """ - with _tempfile.TemporaryDirectory(prefix='nmbug-clone.') as workdir: + with _tempfile.TemporaryDirectory(prefix='notmuch-git-clone.') as workdir: _spawn( args=[ 'git', 'clone', '--no-checkout', '--separate-git-dir', NOTMUCH_GIT_DIR, @@ -357,7 +369,26 @@ class CachedIndex: _git(args=['read-tree', self.current_treeish], wait=True) -def commit(treeish='HEAD', message=None): +def check_safe_fraction(status): + safe = 0.1 + conf = _notmuch_config_get ('git.safe_fraction') + if conf and conf != '': + safe=float(conf) + + total = count_messages (TAG_PREFIX) + if total == 0: + _LOG.error('No existing tags with given prefix, stopping.'.format(safe)) + _LOG.error('Use --force to override.') + exit(1) + change = len(status['added'])+len(status['deleted']) + fraction = change/total + _LOG.debug('total messages {:d}, change: {:d}, fraction: {:f}'.format(total,change,fraction)) + if fraction > safe: + _LOG.error('safe fraction {:f} exceeded, stopping.'.format(safe)) + _LOG.error('Use --force to override or reconfigure git.safe_fraction.') + exit(1) + +def commit(treeish='HEAD', message=None, force=False): """ Commit prefix-matching tags from the notmuch database to Git. """ @@ -368,6 +399,9 @@ def commit(treeish='HEAD', message=None): _LOG.warning('Nothing to commit') return + if not force: + check_safe_fraction (status) + with CachedIndex(NOTMUCH_GIT_DIR, treeish) as index: try: _update_index(status=status) @@ -420,25 +454,40 @@ def fetch(remote=None): def init(remote=None): """ - Create an empty nmbug repository. + Create an empty notmuch-git repository. This wraps 'git init' with a few extra steps to support subsequent status and commit commands. """ + from pathlib import Path + parent = Path(NOTMUCH_GIT_DIR).parent + try: + _os.makedirs(parent) + except FileExistsError: + pass + _spawn(args=['git', '--git-dir', NOTMUCH_GIT_DIR, 'init', '--initial-branch=master', '--quiet', '--bare'], wait=True) _git(args=['config', 'core.logallrefupdates', 'true'], wait=True) # create an empty blob (e69de29bb2d1d6434b8b29ae775ad8c2e48c5391) _git(args=['hash-object', '-w', '--stdin'], input='', wait=True) + # create a blob for the FORMAT file + (status, stdout, _) = _git(args=['hash-object', '-w', '--stdin'], stdout=_subprocess.PIPE, + input='1\n', wait=True) + verhash=stdout.rstrip() + _LOG.debug('hash of FORMAT blob = {:s}'.format(verhash)) + # Add FORMAT to the index + _git(args=['update-index', '--add', '--cacheinfo', '100644,{:s},FORMAT'.format(verhash)], wait=True) + _git( args=[ - 'commit', '--allow-empty', '-m', 'Start a new nmbug repository' + 'commit', '-m', 'Start a new notmuch-git repository' ], additional_env={'GIT_WORK_TREE': NOTMUCH_GIT_DIR}, wait=True) -def checkout(): +def checkout(force=None): """ Update the notmuch database from Git. @@ -446,6 +495,10 @@ def checkout(): to Git. """ status = get_status() + + if not force: + check_safe_fraction(status) + with _spawn( args=['notmuch', 'tag', '--batch'], stdin=_subprocess.PIPE) as p: for id, tags in status['added'].items(): @@ -477,9 +530,9 @@ def _insist_committed(): _LOG.error('\n'.join([ 'Uncommitted changes to {prefix}* tags in notmuch', '', - "For a summary of changes, run 'nmbug status'", - "To save your changes, run 'nmbug commit' before merging/pull", - "To discard your changes, run 'nmbug checkout'", + "For a summary of changes, run 'notmuch-git status'", + "To save your changes, run 'notmuch-git commit' before merging/pull", + "To discard your changes, run 'notmuch-git checkout'", ]).format(prefix=TAG_PREFIX)) _sys.exit(1) @@ -501,7 +554,7 @@ def pull(repository=None, refspecs=None): args.append(repository) if refspecs: args.extend(refspecs) - with _tempfile.TemporaryDirectory(prefix='nmbug-pull.') as workdir: + with _tempfile.TemporaryDirectory(prefix='notmuch-git-pull.') as workdir: for command in [ ['reset', '--hard'], args]: @@ -519,7 +572,7 @@ def merge(reference='@{upstream}'): The default reference is '@{upstream}'. """ _insist_committed() - with _tempfile.TemporaryDirectory(prefix='nmbug-merge.') as workdir: + with _tempfile.TemporaryDirectory(prefix='notmuch-git-merge.') as workdir: for command in [ ['reset', '--hard'], ['merge', reference]]: @@ -534,8 +587,8 @@ def log(args=()): """ A simple wrapper for 'git log'. - After running 'nmbug fetch', you can inspect the changes with - 'nmbug log HEAD..@{upstream}'. + After running 'notmuch-git fetch', you can inspect the changes with + 'notmuch-git log HEAD..@{upstream}'. """ # we don't want output trapping here, because we want the pager. args = ['log', '--name-status', '--no-renames'] + list(args) @@ -544,7 +597,7 @@ def log(args=()): def push(repository=None, refspecs=None): - "Push the local nmbug Git state to a remote repository." + "Push the local notmuch-git Git state to a remote repository." if refspecs and not repository: repository = _get_remote() args = ['push'] @@ -567,13 +620,13 @@ def status(): * A - Tag is present in notmuch database, but not committed to nmbug - (equivalently, tag has been deleted in nmbug repo, e.g. by a + Tag is present in notmuch database, but not committed to notmuch-git + (equivalently, tag has been deleted in notmuch-git repo, e.g. by a pull, but not restored to notmuch database). * D - Tag is present in nmbug repo, but not restored to notmuch + Tag is present in notmuch-git repo, but not restored to notmuch database (equivalently, tag has been deleted in notmuch). * U @@ -581,7 +634,7 @@ def status(): Message is unknown (missing from local notmuch database). The second character (if present) represents a difference between - local and upstream branches. Typically 'nmbug fetch' needs to be + local and upstream branches. Typically 'notmuch-git fetch' needs to be run to update this. * a @@ -778,7 +831,7 @@ def _clear_tags_for_message(index, id): Neither 'id' nor the tags in 'tags' should be encoded/escaped. """ - dir = 'tags/{id}'.format(id=_hex_quote(string=id)) + dir = _id_path(id) with _git( args=['ls-files', dir], @@ -795,6 +848,21 @@ def _read_database_lastmod(): (count,uuid,lastmod_str) = notmuch.stdout.readline().split() return (count,uuid,int(lastmod_str)) +def _id_path(id): + hid=_hex_quote(string=id) + from hashlib import blake2b + + if FORMAT_VERSION==0: + return 'tags/{hid}'.format(hid=hid) + elif FORMAT_VERSION==1: + idhash = blake2b(hid.encode('utf8'), digest_size=2).hexdigest() + return 'tags/{dir1}/{dir2}/{hid}'.format( + hid=hid, + dir1=idhash[0:2],dir2=idhash[2:]) + else: + _LOG.error("Unknown format version",FORMAT_VERSION) + _sys.exit(1) + def _index_tags_for_message(id, status, tags): """ Update the Git index to either create or delete an empty file. @@ -809,8 +877,7 @@ def _index_tags_for_message(id, status, tags): hash = '0000000000000000000000000000000000000000' for tag in tags: - path = 'tags/{id}/{tag}'.format( - id=_hex_quote(string=id), tag=_hex_quote(string=tag)) + path = '{ipath}/{tag}'.format(ipath=_id_path(id),tag=_hex_quote(string=tag)) yield '{mode} {hash}\t{path}\n'.format(mode=mode, hash=hash, path=path) @@ -826,7 +893,7 @@ def _diff_refs(filter, a='HEAD', b='@{upstream}'): def _unpack_diff_lines(stream): "Iterate through (id, tag) tuples in a diff stream." for line in stream: - match = _TAG_FILE_REGEX.match(line.strip()) + match = _TAG_FILE_REGEX[FORMAT_VERSION].match(line.strip()) if not match: message = 'non-tag line in diff: {!r}'.format(line.strip()) if line.startswith(_TAG_DIRECTORY): @@ -840,15 +907,15 @@ def _unpack_diff_lines(stream): def _help(parser, command=None): """ - Show help for an nmbug command. + Show help for an notmuch-git command. Because some folks prefer: - $ nmbug help COMMAND + $ notmuch-git help COMMAND to - $ nmbug COMMAND --help + $ notmuch-git COMMAND --help """ if command: parser.parse_args([command, '--help']) @@ -861,9 +928,30 @@ def _notmuch_config_get(key): stdout=_subprocess.PIPE, wait=True) if status != 0: _LOG.error("failed to run notmuch config") - sys.exit(1) + _sys.exit(1) return stdout.rstrip() +def read_format_version(): + try: + (status, stdout, stderr) = _git( + args=['cat-file', 'blob', 'master:FORMAT'], + stdout=_subprocess.PIPE, stderr=_subprocess.PIPE, wait=True) + except SubprocessError as e: + _LOG.debug("failed to read FORMAT file from git, assuming format version 0") + return 0 + + return int(stdout) + +# based on BaseDirectory.save_data_path from pyxdg (LGPL2+) +def xdg_data_path(profile): + resource = _os.path.join('notmuch',profile,'git') + assert not resource.startswith('/') + _home = _os.path.expanduser('~') + xdg_data_home = _os.environ.get('XDG_DATA_HOME') or \ + _os.path.join(_home, '.local', 'share') + path = _os.path.join(xdg_data_home, resource) + return path + if __name__ == '__main__': import argparse @@ -875,8 +963,11 @@ if __name__ == '__main__': help='Git repository to operate on.') parser.add_argument( '-p', '--tag-prefix', metavar='PREFIX', - default = _os.getenv('NOTMUCH_GIT_PREFIX', 'notmuch::'), + default = None, help='Prefix of tags to operate on.') + parser.add_argument( + '-N', '--nmbug', action='store_true', + help='Set defaults for --tag-prefix and --git-dir for the notmuch bug tracker') parser.add_argument( '-l', '--log-level', choices=['critical', 'error', 'warning', 'info', 'debug'], @@ -923,6 +1014,10 @@ if __name__ == '__main__': help=( "Argument passed through to 'git archive'. Set anything " 'before , see git-archive(1) for details.')) + elif command == 'checkout': + subparser.add_argument( + '-f', '--force', action='store_true', + help='checkout a large fraction of tags.') elif command == 'clone': subparser.add_argument( 'repository', @@ -931,6 +1026,9 @@ if __name__ == '__main__': 'URLS section of git-clone(1) for more information on ' 'specifying repositories.')) elif command == 'commit': + subparser.add_argument( + '-f', '--force', action='store_true', + help='commit a large fraction of tags.') subparser.add_argument( 'message', metavar='MESSAGE', default='', nargs='?', help='Text for the commit message.') @@ -987,16 +1085,38 @@ if __name__ == '__main__': args = parser.parse_args() + nmbug_mode = False + notmuch_profile = _os.getenv('NOTMUCH_PROFILE','default') + + if args.nmbug or _os.path.basename(__file__) == 'nmbug': + nmbug_mode = True + if args.git_dir: NOTMUCH_GIT_DIR = args.git_dir else: - NOTMUCH_GIT_DIR = _os.path.expanduser( - _os.getenv('NOTMUCH_GIT_DIR', _os.path.join('~', '.nmbug'))) - _NOTMUCH_GIT_DIR = _os.path.join(NOTMUCH_GIT_DIR, '.git') - if _os.path.isdir(_NOTMUCH_GIT_DIR): - NOTMUCH_GIT_DIR = _NOTMUCH_GIT_DIR + if nmbug_mode: + default = _os.path.join('~', '.nmbug') + else: + default = _notmuch_config_get ('git.path') + if default == '': + default = xdg_data_path(notmuch_profile) + + NOTMUCH_GIT_DIR = _os.path.expanduser(_os.getenv('NOTMUCH_GIT_DIR', default)) + + _NOTMUCH_GIT_DIR = _os.path.join(NOTMUCH_GIT_DIR, '.git') + if _os.path.isdir(_NOTMUCH_GIT_DIR): + NOTMUCH_GIT_DIR = _NOTMUCH_GIT_DIR + + if args.tag_prefix: + TAG_PREFIX = args.tag_prefix + else: + if nmbug_mode: + prefix = 'notmuch::' + else: + prefix = _notmuch_config_get ('git.tag_prefix') + + TAG_PREFIX = _os.getenv('NOTMUCH_GIT_PREFIX', prefix) - TAG_PREFIX = args.tag_prefix _ENCODED_TAG_PREFIX = _hex_quote(TAG_PREFIX, safe='+@=,') # quote ':' if args.log_level: @@ -1009,12 +1129,19 @@ if __name__ == '__main__': if _notmuch_config_get('built_with.sexp_queries') != 'true': _LOG.error("notmuch git needs sexp query support") - sys.exit(1) + _sys.exit(1) if not getattr(args, 'func', None): parser.print_usage() _sys.exit(1) + # The following two lines are used by the test suite. + _LOG.debug('prefix = {:s}'.format(TAG_PREFIX)) + _LOG.debug('repository = {:s}'.format(NOTMUCH_GIT_DIR)) + + FORMAT_VERSION = read_format_version() + _LOG.debug('FORMAT_VERSION={:d}'.format(FORMAT_VERSION)) + if args.func == help: arg_names = ['command'] else: