X-Git-Url: https://git.cworth.org/git?p=notmuch;a=blobdiff_plain;f=notmuch-git.py;h=ceb86fbc14dfa97db10dfec87732452a648edc0b;hp=aebff76474102311ad8f2515848e40a365c3bb38;hb=HEAD;hpb=6219e7380ae34cc0c8142f4174bee3cde9bf9662 diff --git a/notmuch-git.py b/notmuch-git.py index aebff764..ee87bec6 100644 --- a/notmuch-git.py +++ b/notmuch-git.py @@ -31,7 +31,6 @@ import locale as _locale import logging as _logging import os as _os import re as _re -import shutil as _shutil import subprocess as _subprocess import sys as _sys import tempfile as _tempfile @@ -46,7 +45,7 @@ _LOG.addHandler(_logging.StreamHandler()) NOTMUCH_GIT_DIR = None TAG_PREFIX = None -FORMAT_VERSION = 0 +FORMAT_VERSION = 1 _HEX_ESCAPE_REGEX = _re.compile('%[0-9A-F]{2}') _TAG_DIRECTORY = 'tags/' @@ -248,13 +247,13 @@ def count_messages(prefix=None): stdout=_subprocess.PIPE, wait=True) if status != 0: _LOG.error("failed to run notmuch config") - sys.exit(1) + _sys.exit(1) return int(stdout.rstrip()) def get_tags(prefix=None): "Get a list of tags with a given prefix." (status, stdout, stderr) = _spawn( - args=['notmuch', 'search', '--query=sexp', '--output=tags', _tag_query(prefix)], + args=['notmuch', 'search', '--exclude=false', '--query=sexp', '--output=tags', _tag_query(prefix)], stdout=_subprocess.PIPE, wait=True) return [tag for tag in stdout.splitlines()] @@ -369,7 +368,7 @@ class CachedIndex: _git(args=['read-tree', self.current_treeish], wait=True) -def check_safe_fraction(status): +def _check_fraction(change): safe = 0.1 conf = _notmuch_config_get ('git.safe_fraction') if conf and conf != '': @@ -377,10 +376,10 @@ def check_safe_fraction(status): total = count_messages (TAG_PREFIX) if total == 0: - _LOG.error('No existing tags with given prefix, stopping.'.format(safe)) + _LOG.error('No existing tags with given prefix, stopping.') _LOG.error('Use --force to override.') exit(1) - change = len(status['added'])+len(status['deleted']) + fraction = change/total _LOG.debug('total messages {:d}, change: {:d}, fraction: {:f}'.format(total,change,fraction)) if fraction > safe: @@ -388,6 +387,25 @@ def check_safe_fraction(status): _LOG.error('Use --force to override or reconfigure git.safe_fraction.') exit(1) +def check_safe_fraction(status): + + change = len(status['added'])+len(status['deleted']) + _check_fraction(change) + +def check_diff_fraction(): + + # check number of directories (i.e. messages) changed. + change_set = set() + + with _git(args=['diff', '--name-only', 'HEAD', '@{upstream}'], + stdout=_subprocess.PIPE) as git: + for path in git.stdout: + change_set.add(_os.path.dirname(path)) + + change=len(change_set) + _check_fraction(change) + + def commit(treeish='HEAD', message=None, force=False): """ Commit prefix-matching tags from the notmuch database to Git. @@ -452,7 +470,7 @@ def fetch(remote=None): _git(args=args, wait=True) -def init(remote=None): +def init(remote=None,format_version=None): """ Create an empty notmuch-git repository. @@ -466,22 +484,34 @@ def init(remote=None): except FileExistsError: pass + if not format_version: + format_version = 1 + + format_version=int(format_version) + + if format_version > 1 or format_version < 0: + _LOG.error("Illegal format version {:d}".format(format_version)) + _sys.exit(1) + _spawn(args=['git', '--git-dir', NOTMUCH_GIT_DIR, 'init', '--initial-branch=master', '--quiet', '--bare'], wait=True) _git(args=['config', 'core.logallrefupdates', 'true'], wait=True) # create an empty blob (e69de29bb2d1d6434b8b29ae775ad8c2e48c5391) _git(args=['hash-object', '-w', '--stdin'], input='', wait=True) - # create a blob for the FORMAT file - (status, stdout, _) = _git(args=['hash-object', '-w', '--stdin'], stdout=_subprocess.PIPE, - input='1\n', wait=True) - verhash=stdout.rstrip() - _LOG.debug('hash of FORMAT blob = {:s}'.format(verhash)) - # Add FORMAT to the index - _git(args=['update-index', '--add', '--cacheinfo', '100644,{:s},FORMAT'.format(verhash)], wait=True) + allow_empty=('--allow-empty',) + if format_version >= 1: + allow_empty=() + # create a blob for the FORMAT file + (status, stdout, _) = _git(args=['hash-object', '-w', '--stdin'], stdout=_subprocess.PIPE, + input='{:d}\n'.format(format_version), wait=True) + verhash=stdout.rstrip() + _LOG.debug('hash of FORMAT blob = {:s}'.format(verhash)) + # Add FORMAT to the index + _git(args=['update-index', '--add', '--cacheinfo', '100644,{:s},FORMAT'.format(verhash)], wait=True) _git( args=[ - 'commit', '-m', 'Start a new notmuch-git repository' + 'commit', *allow_empty, '-m', 'Start a new notmuch-git repository' ], additional_env={'GIT_WORK_TREE': NOTMUCH_GIT_DIR}, wait=True) @@ -608,6 +638,15 @@ def push(repository=None, refspecs=None): _git(args=args, wait=True) +def reset(force=False): + """ + reset the local git branch to match the remote one + """ + if not force: + check_diff_fraction() + + _git(args=["reset","--soft","origin/master"],wait=True) + def status(): """ Show pending updates in notmuch or git repo. @@ -686,6 +725,32 @@ def _is_unmerged(ref='@{upstream}'): stdout=_subprocess.PIPE, wait=True) return base != fetch_head +class DatabaseCache: + def __init__(self): + try: + from notmuch2 import Database + self._notmuch = Database() + except ImportError: + self._notmuch = None + self._known = {} + + def known(self,id): + if id in self._known: + return self._known[id]; + + if self._notmuch: + try: + _ = self._notmuch.find(id) + self._known[id] = True + except LookupError: + self._known[id] = False + else: + (_, stdout, stderr) = _spawn( + args=['notmuch', 'search', '--exclude=false', '--output=files', 'id:{0}'.format(id)], + stdout=_subprocess.PIPE, + wait=True) + self._known[id] = stdout != None + return self._known[id] @timed def get_status(): @@ -693,14 +758,11 @@ def get_status(): 'deleted': {}, 'missing': {}, } + db = DatabaseCache() with PrivateIndex(repo=NOTMUCH_GIT_DIR, prefix=TAG_PREFIX) as index: maybe_deleted = index.diff(filter='D') for id, tags in maybe_deleted.items(): - (_, stdout, stderr) = _spawn( - args=['notmuch', 'search', '--output=files', 'id:{0}'.format(id)], - stdout=_subprocess.PIPE, - wait=True) - if stdout: + if db.known(id): status['deleted'][id] = tags else: status['missing'][id] = tags @@ -726,6 +788,7 @@ class PrivateIndex: self.lastmod = None self.checksum = None self._load_cache_file() + self.file_tree = None self._index_tags() def __enter__(self): @@ -751,6 +814,43 @@ class PrivateIndex: _LOG.error("Error decoding cache") _sys.exit(1) + @timed + def _read_file_tree(self): + self.file_tree = {} + + with _git( + args=['ls-files', 'tags'], + additional_env={'GIT_INDEX_FILE': self.index_path}, + stdout=_subprocess.PIPE) as git: + for file in git.stdout: + dir=_os.path.dirname(file) + tag=_os.path.basename(file).rstrip() + if dir not in self.file_tree: + self.file_tree[dir]=[tag] + else: + self.file_tree[dir].append(tag) + + + def _clear_tags_for_message(self, id): + """ + Clear any existing index entries for message 'id' + + Neither 'id' nor the tags in 'tags' should be encoded/escaped. + """ + + if self.file_tree == None: + self._read_file_tree() + + dir = _id_path(id) + + if dir not in self.file_tree: + return + + for file in self.file_tree[dir]: + line = '0 0000000000000000000000000000000000000000\t{:s}/{:s}\n'.format(dir,file) + yield line + + @timed def _index_tags(self): "Write notmuch tags to private git index." @@ -786,7 +886,7 @@ class PrivateIndex: if tag.startswith(prefix)] id = _xapian_unquote(string=id) if clear_tags: - for line in _clear_tags_for_message(index=self.index_path, id=id): + for line in self._clear_tags_for_message(id=id): git.stdin.write(line) for line in _index_tags_for_message( id=id, status='A', tags=tags): @@ -823,24 +923,6 @@ def _read_index_checksum (index_path): except FileNotFoundError: return None - -def _clear_tags_for_message(index, id): - """ - Clear any existing index entries for message 'id' - - Neither 'id' nor the tags in 'tags' should be encoded/escaped. - """ - - dir = _id_path(id) - - with _git( - args=['ls-files', dir], - additional_env={'GIT_INDEX_FILE': index}, - stdout=_subprocess.PIPE) as git: - for file in git.stdout: - line = '0 0000000000000000000000000000000000000000\t{:s}\n'.format(file.strip()) - yield line - def _read_database_lastmod(): with _spawn( args=['notmuch', 'count', '--lastmod', '*'], @@ -993,6 +1075,7 @@ if __name__ == '__main__': 'merge', 'pull', 'push', + 'reset', 'status', ]: func = locals()[command] @@ -1043,6 +1126,11 @@ if __name__ == '__main__': subparser.add_argument( 'command', metavar='COMMAND', nargs='?', help='The command to show help for.') + elif command == 'init': + subparser.add_argument( + '--format-version', metavar='VERSION', + default = None, + help='create format VERSION repository.') elif command == 'log': subparser.add_argument( 'args', metavar='ARG', nargs='*', @@ -1082,6 +1170,10 @@ if __name__ == '__main__': 'Refspec (usually a branch name) to push. See ' 'the entry in the OPTIONS section of ' 'git-push(1) for other possibilities.')) + elif command == 'reset': + subparser.add_argument( + '-f', '--force', action='store_true', + help='reset a large fraction of tags.') args = parser.parse_args() @@ -1139,7 +1231,9 @@ if __name__ == '__main__': _LOG.debug('prefix = {:s}'.format(TAG_PREFIX)) _LOG.debug('repository = {:s}'.format(NOTMUCH_GIT_DIR)) - FORMAT_VERSION = read_format_version() + if args.func != init: + FORMAT_VERSION = read_format_version() + _LOG.debug('FORMAT_VERSION={:d}'.format(FORMAT_VERSION)) if args.func == help: