From: David Bremner Date: Sat, 30 Apr 2022 17:26:55 +0000 (-0300) Subject: Merge tag 'debian/0.36-1' into debian/bullseye-backports X-Git-Tag: archive/debian/0.36-1_bpo11+1~1 X-Git-Url: https://git.cworth.org/git?a=commitdiff_plain;h=97b6a43d46be19b73c2848b29184a93db06ddfe8;hp=4a4ea3234e6bd056aaa4b826765c089e8c884882;p=notmuch Merge tag 'debian/0.36-1' into debian/bullseye-backports notmuch release 0.36-1 for unstable (sid) [dgit] [dgit distro=debian no-split --quilt=linear] --- diff --git a/Makefile.local b/Makefile.local index 10fb9908..d8bbf3e1 100644 --- a/Makefile.local +++ b/Makefile.local @@ -73,7 +73,7 @@ release: verify-source-tree-and-version ln -sf $(TAR_FILE) $(DEB_TAR_FILE) pristine-tar commit $(DEB_TAR_FILE) $(UPSTREAM_TAG) mkdir -p releases - mv $(TAR_FILE) $(SHA256_FILE) $(DETACHED_SIG_FILE) releases + mv $(TAR_FILE) $(DEB_TAR_FILE) $(SHA256_FILE) $(DETACHED_SIG_FILE) releases $(MAKE) VERSION=$(VERSION) release-message > $(PACKAGE)-$(VERSION).announce ifeq ($(REALLY_UPLOAD),yes) git push origin $(VERSION) release pristine-tar @@ -92,7 +92,11 @@ pre-release: ln -sf $(TAR_FILE) $(DEB_TAR_FILE) pristine-tar commit $(DEB_TAR_FILE) $(UPSTREAM_TAG) mkdir -p releases - mv $(TAR_FILE) $(DEB_TAR_FILE) releases + mv $(TAR_FILE) $(DEB_TAR_FILE) $(SHA256_FILE) $(DETACHED_SIG_FILE) releases +ifeq ($(REALLY_UPLOAD),yes) + git push origin $(UPSTREAM_TAG) release pristine-tar + cd releases && scp $(TAR_FILE) $(SHA256_FILE) $(DETACHED_SIG_FILE) $(RELEASE_HOST):$(RELEASE_DIR) +endif .PHONY: debian-snapshot debian-snapshot: @@ -120,8 +124,7 @@ release-message: @echo "Which can be verified with:" @echo "" @echo " $(RELEASE_URL)/$(SHA256_FILE)" - @echo -n " " - @cat releases/$(SHA256_FILE) + @sed "s/^/ /" releases/$(SHA256_FILE) @echo "" @echo " $(RELEASE_URL)/$(DETACHED_SIG_FILE)" @echo " (signed by `getent passwd "$$USER" | cut -d: -f 5 | cut -d, -f 1`)" @@ -169,7 +172,7 @@ release-checks: .PHONY: verify-newer verify-newer: - @echo -n "Checking that no $(VERSION) release already exists..." + @printf %s "Checking that no $(VERSION) release already exists..." @wget -q --no-check-certificate -O /dev/null $(RELEASE_URL)/$(TAR_FILE) ; \ case $$? in \ 8) echo "Good." ;; \ diff --git a/NEWS b/NEWS index c6ce2eea..c18d63d3 100644 --- a/NEWS +++ b/NEWS @@ -1,3 +1,60 @@ +Notmuch 0.36 (2022-04-25) +========================= + +Library +------- + +Add the `sexp` prefix to the infix (traditional) query parser. This +allows specific subqueries to be parsed by the sexp parser (with +appropropriate quoting). See `notmuch-search-terms(7)` for details. + +Add another heuristic to regexp fields to prevent phrase parsing of +bracketed sub-expressions. + +Command Line Interface +---------------------- + +Envelope from ("From ") headers are now escaped as X-Envelope-From: in +input to `notmuch-insert`. This prevents creating mbox files when +calling `notmuch-insert` from e.g. `postfix`. + +Python (CFFI) Bindings +---------------------- + +Use the `config_pairs` API in ConfigIterator. This returns all +matching key-value pairs, not just those that happen to be stored in +the database. + +Documentation +------------- + +Reorganize documention for `notmuch-config`. Add a few links from +other man pages. + +Emacs +----- + +Bind the usual undo key sequences to new command +"notmuch-tag-undo". This allows transparent undo of tagging +operations. + +Tests +----- + +Fix smime.4 with newer gmime. Unset `XDG_DATA_HOME` and `MAILDIR` for tests. + +New add-on tool: notmuch-web +----------------------------- + +The new devel/ tool `notmuch-web` is a very thin web client. It +supports a full search interface for one user: there is no facility +for multiple users provided today. See the notmuch-web README file +for more information. + +Be careful about running it on a network-connected system: it will +expose a web interface that requires no authentication but exposes +your mail store. + Notmuch 0.35 (2022-02-06) ========================= diff --git a/bindings/python-cffi/notmuch2/_build.py b/bindings/python-cffi/notmuch2/_build.py index a55b484f..349bb79d 100644 --- a/bindings/python-cffi/notmuch2/_build.py +++ b/bindings/python-cffi/notmuch2/_build.py @@ -97,7 +97,7 @@ ffibuilder.cdef( typedef struct _notmuch_string_map_iterator notmuch_message_properties_t; typedef struct _notmuch_directory notmuch_directory_t; typedef struct _notmuch_filenames notmuch_filenames_t; - typedef struct _notmuch_config_list notmuch_config_list_t; + typedef struct _notmuch_config_pairs notmuch_config_pairs_t; typedef struct _notmuch_indexopts notmuch_indexopts_t; const char * @@ -325,18 +325,18 @@ ffibuilder.cdef( notmuch_database_set_config (notmuch_database_t *db, const char *key, const char *value); notmuch_status_t notmuch_database_get_config (notmuch_database_t *db, const char *key, char **value); - notmuch_status_t - notmuch_database_get_config_list (notmuch_database_t *db, const char *prefix, notmuch_config_list_t **out); + notmuch_config_pairs_t * + notmuch_config_get_pairs (notmuch_database_t *db, const char *prefix); notmuch_bool_t - notmuch_config_list_valid (notmuch_config_list_t *config_list); + notmuch_config_pairs_valid (notmuch_config_pairs_t *config_list); const char * - notmuch_config_list_key (notmuch_config_list_t *config_list); + notmuch_config_pairs_key (notmuch_config_pairs_t *config_list); const char * - notmuch_config_list_value (notmuch_config_list_t *config_list); + notmuch_config_pairs_value (notmuch_config_pairs_t *config_list); void - notmuch_config_list_move_to_next (notmuch_config_list_t *config_list); + notmuch_config_pairs_move_to_next (notmuch_config_pairs_t *config_list); void - notmuch_config_list_destroy (notmuch_config_list_t *config_list); + notmuch_config_pairs_destroy (notmuch_config_pairs_t *config_list); """ ) diff --git a/bindings/python-cffi/notmuch2/_config.py b/bindings/python-cffi/notmuch2/_config.py index 29de6495..603fdcbf 100644 --- a/bindings/python-cffi/notmuch2/_config.py +++ b/bindings/python-cffi/notmuch2/_config.py @@ -13,27 +13,42 @@ class ConfigIter(base.NotmuchIter): def __init__(self, parent, iter_p): super().__init__( parent, iter_p, - fn_destroy=capi.lib.notmuch_config_list_destroy, - fn_valid=capi.lib.notmuch_config_list_valid, - fn_get=capi.lib.notmuch_config_list_key, - fn_next=capi.lib.notmuch_config_list_move_to_next) + fn_destroy=capi.lib.notmuch_config_pairs_destroy, + fn_valid=capi.lib.notmuch_config_pairs_valid, + fn_get=capi.lib.notmuch_config_pairs_key, + fn_next=capi.lib.notmuch_config_pairs_move_to_next) def __next__(self): - item = super().__next__() - return base.BinString.from_cffi(item) - + # skip pairs whose value is NULL + while capi.lib.notmuch_config_pairs_valid(super()._iter_p): + val_p = capi.lib.notmuch_config_pairs_value(super()._iter_p) + key_p = capi.lib.notmuch_config_pairs_key(super()._iter_p) + if key_p == capi.ffi.NULL: + # this should never happen + raise errors.NullPointerError + key = base.BinString.from_cffi(key_p) + capi.lib.notmuch_config_pairs_move_to_next(super()._iter_p) + if val_p != capi.ffi.NULL and base.BinString.from_cffi(val_p) != "": + return key + self._destroy() + raise StopIteration class ConfigMapping(base.NotmuchObject, collections.abc.MutableMapping): - """The config key/value pairs stored in the database. + """The config key/value pairs loaded from the database, config file, + and and/or defaults. The entries are exposed as a :class:`collections.abc.MutableMapping` object. Note that setting a value to an empty string is the same as deleting it. + Mutating (deleting or updating values) in the map persists only in + the database, which can be shadowed by config files. + :param parent: the parent object :param ptr_name: the name of the attribute on the parent which will return the memory pointer. This allows this object to access the pointer via the parent's descriptor and thus trigger :class:`MemoryPointer`'s memory safety. + """ def __init__(self, parent, ptr_name): @@ -77,11 +92,10 @@ class ConfigMapping(base.NotmuchObject, collections.abc.MutableMapping): :raises NullPointerError: If the iterator can not be created. """ - configlist_pp = capi.ffi.new('notmuch_config_list_t**') - ret = capi.lib.notmuch_database_get_config_list(self._ptr(), b'', configlist_pp) - if ret != capi.lib.NOTMUCH_STATUS_SUCCESS: - raise errors.NotmuchError(ret) - return ConfigIter(self._parent, configlist_pp[0]) + config_pairs_p = capi.lib.notmuch_config_get_pairs(self._ptr(), b'') + if config_pairs_p == capi.ffi.NULL: + raise KeyError + return ConfigIter(self._parent, config_pairs_p) def __len__(self): return sum(1 for t in self) diff --git a/bindings/python-cffi/tests/test_config.py b/bindings/python-cffi/tests/test_config.py index 67b0dea4..2a7f42f0 100644 --- a/bindings/python-cffi/tests/test_config.py +++ b/bindings/python-cffi/tests/test_config.py @@ -34,20 +34,24 @@ class TestIter: print(repr(val)) def test_iter(self, db): - assert list(db.config) == [] - db.config['spam'] = 'ham' - db.config['eggs'] = 'bacon' - assert set(db.config) == {'spam', 'eggs'} - assert set(db.config.keys()) == {'spam', 'eggs'} - assert set(db.config.values()) == {'ham', 'bacon'} - assert set(db.config.items()) == {('spam', 'ham'), ('eggs', 'bacon')} + def has_prefix(x): + return x.startswith('TEST.') + + assert [ x for x in db.config if has_prefix(x) ] == [] + db.config['TEST.spam'] = 'TEST.ham' + db.config['TEST.eggs'] = 'TEST.bacon' + assert { x for x in db.config if has_prefix(x) } == {'TEST.spam', 'TEST.eggs'} + assert { x for x in db.config.keys() if has_prefix(x) } == {'TEST.spam', 'TEST.eggs'} + assert { x for x in db.config.values() if has_prefix(x) } == {'TEST.ham', 'TEST.bacon'} + assert { (x, y) for (x,y) in db.config.items() if has_prefix(x) } == \ + {('TEST.spam', 'TEST.ham'), ('TEST.eggs', 'TEST.bacon')} def test_len(self, db): - assert len(db.config) == 0 + defaults = len(db.config) db.config['spam'] = 'ham' - assert len(db.config) == 1 + assert len(db.config) == defaults + 1 db.config['eggs'] = 'bacon' - assert len(db.config) == 2 + assert len(db.config) == defaults + 2 def test_del(self, db): db.config['spam'] = 'ham' diff --git a/bindings/python/notmuch/version.py b/bindings/python/notmuch/version.py index f7392174..59144626 100644 --- a/bindings/python/notmuch/version.py +++ b/bindings/python/notmuch/version.py @@ -1,3 +1,3 @@ # this file should be kept in sync with ../../../version -__VERSION__ = '0.35' +__VERSION__ = '0.36' SOVERSION = '5' diff --git a/configure b/configure index 36f3f606..2f6d8b68 100755 --- a/configure +++ b/configure @@ -552,11 +552,7 @@ EOF rm -rf "$TEMP_GPG" fi - # see https://github.com/jstedfast/gmime/pull/90 - # should be fixed in GMime in 3.2.7, but some distros might patch - printf "Checking for GMime X.509 certificate validity... " - - cat > _check_x509_validity.c < _check_gmime_cert.c < #include @@ -589,16 +585,27 @@ int main () { if (sig == NULL) return !! fprintf (stderr, "no GMimeSignature found at position 0\n"); cert = g_mime_signature_get_certificate (sig); if (cert == NULL) return !! fprintf (stderr, "no GMimeCertificate found\n"); +#ifdef CHECK_VALIDITY validity = g_mime_certificate_get_id_validity (cert); if (validity != GMIME_VALIDITY_FULL) return !! fprintf (stderr, "Got validity %d, expected %d\n", validity, GMIME_VALIDITY_FULL); - +#endif +#ifdef CHECK_EMAIL + const char *email = g_mime_certificate_get_email (cert); + if (! email) return !! fprintf (stderr, "no email returned"); + if (email[0] == '<') return 2; +#endif return 0; } EOF + + # see https://github.com/jstedfast/gmime/pull/90 + # should be fixed in GMime in 3.2.7, but some distros might patch + printf "Checking for GMime X.509 certificate validity... " + if ! TEMP_GPG=$(mktemp -d "${TMPDIR:-/tmp}/notmuch.XXXXXX"); then printf 'No.\nCould not make tempdir for testing X.509 certificate validity support.\n' errors=$((errors + 1)) - elif ${CC} ${CFLAGS} ${gmime_cflags} _check_x509_validity.c ${gmime_ldflags} -o _check_x509_validity \ + elif ${CC} -DCHECK_VALIDITY ${CFLAGS} ${gmime_cflags} _check_gmime_cert.c ${gmime_ldflags} -o _check_x509_validity \ && echo disable-crl-checks > "$TEMP_GPG/gpgsm.conf" \ && echo "4D:E0:FF:63:C0:E9:EC:01:29:11:C8:7A:EE:DA:3A:9A:7F:6E:C1:0D S" >> "$TEMP_GPG/trustlist.txt" \ && GNUPGHOME=${TEMP_GPG} gpgsm --batch --quiet --import < "$srcdir"/test/smime/ca.crt @@ -620,6 +627,15 @@ EOF errors=$((errors + 1)) fi fi + printf "Checking whether GMime emits email addresses with angle brackets... " + if ${CC} -DCHECK_EMAIL ${CFLAGS} ${gmime_cflags} _check_gmime_cert.c ${gmime_ldflags} -o _check_email && + GNUPGHOME=${TEMP_GPG} ./_check_email; then + gmime_emits_angle_brackets=0 + printf "No.\n" + else + gmime_emits_angle_brackets=1 + printf "Yes.\n" + fi else printf 'No.\nFailed to set up gpgsm for testing X.509 certificate validity support.\n' errors=$((errors + 1)) @@ -1256,7 +1272,7 @@ for flag in -Wmissing-declarations; do done printf "\n\t%s\n" "${WARN_CFLAGS}" -rm -f minimal minimal.c _time_t.c _libversion.c _libversion _libversion.sh _check_session_keys.c _check_session_keys _check_x509_validity.c _check_x509_validity \ +rm -f minimal minimal.c _time_t.c _libversion.c _libversion _libversion.sh _check_session_keys.c _check_session_keys _check_gmime_cert.c _check_x509_validity _check_email \ _verify_sig_with_session_key.c _verify_sig_with_session_key # construct the Makefile.config @@ -1557,6 +1573,9 @@ NOTMUCH_HAVE_XAPIAN_DB_RETRY_LOCK=${WITH_RETRY_LOCK} # Whether GMime can verify X.509 certificate validity NOTMUCH_GMIME_X509_CERT_VALIDITY=${gmime_x509_cert_validity} +# Whether GMime emits addresses with angle brackets (with <>) +NOTMUCH_GMIME_EMITS_ANGLE_BRACKETS=${gmime_emits_angle_brackets} + # Whether GMime can verify signatures when decrypting with a session key: NOTMUCH_GMIME_VERIFY_WITH_SESSION_KEY=${gmime_verify_with_session_key} diff --git a/debian/changelog b/debian/changelog index c938bf3a..b2bff7f3 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,31 @@ +notmuch (0.36-1) unstable; urgency=medium + + * New upstream release + + -- David Bremner Mon, 25 Apr 2022 08:47:41 -0300 + +notmuch (0.36~rc1-1) experimental; urgency=medium + + * New upstream release candidate + * Fix for build in environments where libsexp is not available + (i.e. outside Debian). + + -- David Bremner Sat, 16 Apr 2022 08:37:12 -0300 + +notmuch (0.36~rc0-1) experimental; urgency=medium + + * New upstream release candidate + * Re-enable test smime.4, allegedly fixed upstream. + + -- David Bremner Fri, 15 Apr 2022 08:45:10 -0300 + +notmuch (0.35-2) unstable; urgency=medium + + * Disable test smime.4, which is broken by gmime 3.2.9 thanks to Lucas + Nussbaum for the report (Closes: #1008462). + + -- David Bremner Mon, 28 Mar 2022 11:45:11 -0600 + notmuch (0.35-1~bpo11+2) bullseye-backports; urgency=medium * Once more with binaries diff --git a/debian/rules b/debian/rules index 55867126..f9196b82 100755 --- a/debian/rules +++ b/debian/rules @@ -15,9 +15,6 @@ override_dh_auto_configure: --zshcompletiondir=/usr/share/zsh/vendor-completions \ --localstatedir=/var -override_dh_auto_test: - dh_auto_test -- V=1 - override_dh_auto_build: dh_auto_build -- V=1 all sphinx-html PYBUILD_NAME=notmuch dh_auto_build --buildsystem=pybuild --sourcedirectory bindings/python diff --git a/devel/emacs-keybindings.org b/devel/emacs-keybindings.org index 2f73a198..00977bc3 100644 --- a/devel/emacs-keybindings.org +++ b/devel/emacs-keybindings.org @@ -1,58 +1,59 @@ -|-----------+----------------------------------------+-------------------------------------------------------+-----------------------------------------| -| Key | Search Mode | Show Mode | Tree Mode | -|-----------+----------------------------------------+-------------------------------------------------------+-----------------------------------------| -| a | notmuch-search-archive-thread | notmuch-show-archive-message-then-next-or-next-thread | notmuch-tree-archive-message-then-next | -| b | notmuch-search-scroll-down | notmuch-show-resend-message | notmuch-show-resend-message | -| c | notmuch-search-stash-map | notmuch-show-stash-map | notmuch-show-stash-map | -| d | | | | -| e | | | (notmuch-tree-button-activate) | -| f | | notmuch-show-forward-message | notmuch-show-forward-message | -| g | | | | -| h | | notmuch-show-toggle-visibility-headers | | -| i | | | | -| j | notmuch-jump-search | notmuch-jump-search | notmuch-jump-search | -| k | notmuch-tag-jump | notmuch-tag-jump | notmuch-tag-jump | -| l | notmuch-search-filter | notmuch-show-filter-thread | notmuch-tree-filter | -| m | notmuch-mua-new-mail | notmuch-mua-new-mail | notmuch-mua-new-mail | -| n | notmuch-search-next-thread | notmuch-show-next-open-message | notmuch-tree-next-matching-message | -| o | notmuch-search-toggle-order | | notmuch-tree-toggle-order | -| p | notmuch-search-previous-thread | notmuch-show-previous-open-message | notmuch-tree-prev-matching-message | -| q | notmuch-bury-or-kill-this-buffer | notmuch-bury-or-kill-this-buffer | notmuch-bury-or-kill-this-buffer | -| r | notmuch-search-reply-to-thread-sender | notmuch-show-reply-sender | notmuch-show-reply-sender | -| s | notmuch-search | notmuch-search | notmuch-search | -| t | notmuch-search-filter-by-tag | toggle-truncate-lines | notmuch-tree-filter-by-tag | -| u | | | | -| v | | | notmuch-show-view-all-mime-parts | -| w | | notmuch-show-save-attachments | notmuch-show-save-attachments | -| x | notmuch-bury-or-kill-this-buffer | notmuch-show-archive-message-then-next-or-exit | notmuch-tree-quit | -| y | | | | -| z | notmuch-tree | notmuch-tree | notmuch-tree-to-tree | -| A | | notmuch-show-archive-thread-then-next | notmuch-tree-archive-thread | -| F | | notmuch-show-forward-open-messages | | -| G | notmuch-poll-and-refresh-this-buffer | notmuch-poll-and-refresh-this-buffer | notmuch-poll-and-refresh-this-buffer | -| N | | notmuch-show-next-message | notmuch-tree-next-message | -| O | | | | -| P | | notmuch-show-previous-message | notmuch-tree-prev-message | -| R | notmuch-search-reply-to-thread | notmuch-show-reply | notmuch-show-reply | -| S | | | notmuch-search-from-tree-current-query | -| V | | notmuch-show-view-raw-message | notmuch-show-view-raw-message | -| X | | notmuch-show-archive-thread-then-exit | | -| Z | notmuch-tree-from-search-current-query | notmuch-tree-from-show-current-query | | -| =!= | | notmuch-show-toggle-elide-non-matching | | -| =#= | | notmuch-show-print-message | | -| =$= | | notmuch-show-toggle-process-crypto | | -| =*= | notmuch-search-tag-all | notmuch-show-tag-all | notmuch-tree-tag-thread | -| + | notmuch-search-add-tag | notmuch-show-add-tag | notmuch-tree-add-tag | -| - | notmuch-search-remove-tag | notmuch-show-remove-tag | notmuch-tree-remove-tag | -| . | | notmuch-show-part-map | | -| < | notmuch-search-first-thread | notmuch-show-toggle-thread-indentation | | -| | notmuch-search-scroll-down | notmuch-show-rewind | notmuch-tree-scroll-message-window-back | -| | notmuch-search-show-thread | notmuch-show-toggle-message | notmuch-tree-show-message | -| | notmuch-search-scroll-up | notmuch-show-advance | notmuch-tree-scroll-or-next | -| | | notmuch-show-next-button | notmuch-show-next-button | -| | | notmuch-show-previous-button | notmuch-show-previous-button | -| = | notmuch-refresh-this-buffer | notmuch-refresh-this-buffer | notmuch-tree-refresh-view | -| > | notmuch-search-last-thread | | | -| ? | notmuch-help | notmuch-help | notmuch-help | -| \vert | | notmuch-show-pipe-message | notmuch-show-pipe-message | -|-----------+----------------------------------------+-------------------------------------------------------+-----------------------------------------| +|--------------+----------------------------------------+-------------------------------------------------------+-----------------------------------------| +| Key | Search Mode | Show Mode | Tree Mode | +|--------------+----------------------------------------+-------------------------------------------------------+-----------------------------------------| +| a | notmuch-search-archive-thread | notmuch-show-archive-message-then-next-or-next-thread | notmuch-tree-archive-message-then-next | +| b | notmuch-search-scroll-down | notmuch-show-resend-message | notmuch-show-resend-message | +| c | notmuch-search-stash-map | notmuch-show-stash-map | notmuch-show-stash-map | +| d | | | | +| e | | | (notmuch-tree-button-activate) | +| f | | notmuch-show-forward-message | notmuch-show-forward-message | +| g | | | | +| h | | notmuch-show-toggle-visibility-headers | | +| i | | | | +| j | notmuch-jump-search | notmuch-jump-search | notmuch-jump-search | +| k | notmuch-tag-jump | notmuch-tag-jump | notmuch-tag-jump | +| l | notmuch-search-filter | notmuch-show-filter-thread | notmuch-tree-filter | +| m | notmuch-mua-new-mail | notmuch-mua-new-mail | notmuch-mua-new-mail | +| n | notmuch-search-next-thread | notmuch-show-next-open-message | notmuch-tree-next-matching-message | +| o | notmuch-search-toggle-order | | notmuch-tree-toggle-order | +| p | notmuch-search-previous-thread | notmuch-show-previous-open-message | notmuch-tree-prev-matching-message | +| q | notmuch-bury-or-kill-this-buffer | notmuch-bury-or-kill-this-buffer | notmuch-bury-or-kill-this-buffer | +| r | notmuch-search-reply-to-thread-sender | notmuch-show-reply-sender | notmuch-show-reply-sender | +| s | notmuch-search | notmuch-search | notmuch-search | +| t | notmuch-search-filter-by-tag | toggle-truncate-lines | notmuch-tree-filter-by-tag | +| u | | | | +| v | | | notmuch-show-view-all-mime-parts | +| w | | notmuch-show-save-attachments | notmuch-show-save-attachments | +| x | notmuch-bury-or-kill-this-buffer | notmuch-show-archive-message-then-next-or-exit | notmuch-tree-quit | +| y | | | | +| z | notmuch-tree | notmuch-tree | notmuch-tree-to-tree | +| A | | notmuch-show-archive-thread-then-next | notmuch-tree-archive-thread | +| F | | notmuch-show-forward-open-messages | | +| G | notmuch-poll-and-refresh-this-buffer | notmuch-poll-and-refresh-this-buffer | notmuch-poll-and-refresh-this-buffer | +| N | | notmuch-show-next-message | notmuch-tree-next-message | +| O | | | | +| P | | notmuch-show-previous-message | notmuch-tree-prev-message | +| R | notmuch-search-reply-to-thread | notmuch-show-reply | notmuch-show-reply | +| S | | | notmuch-search-from-tree-current-query | +| V | | notmuch-show-view-raw-message | notmuch-show-view-raw-message | +| X | | notmuch-show-archive-thread-then-exit | | +| Z | notmuch-tree-from-search-current-query | notmuch-tree-from-show-current-query | | +| =!= | | notmuch-show-toggle-elide-non-matching | | +| =#= | | notmuch-show-print-message | | +| =$= | | notmuch-show-toggle-process-crypto | | +| =*= | notmuch-search-tag-all | notmuch-show-tag-all | notmuch-tree-tag-thread | +| + | notmuch-search-add-tag | notmuch-show-add-tag | notmuch-tree-add-tag | +| - | notmuch-search-remove-tag | notmuch-show-remove-tag | notmuch-tree-remove-tag | +| . | | notmuch-show-part-map | | +| < | notmuch-search-first-thread | notmuch-show-toggle-thread-indentation | | +| | notmuch-search-scroll-down | notmuch-show-rewind | notmuch-tree-scroll-message-window-back | +| | notmuch-search-show-thread | notmuch-show-toggle-message | notmuch-tree-show-message | +| | notmuch-search-scroll-up | notmuch-show-advance | notmuch-tree-scroll-or-next | +| | | notmuch-show-next-button | notmuch-show-next-button | +| | | notmuch-show-previous-button | notmuch-show-previous-button | +| = | notmuch-refresh-this-buffer | notmuch-refresh-this-buffer | notmuch-tree-refresh-view | +| > | notmuch-search-last-thread | | | +| ? | notmuch-help | notmuch-help | notmuch-help | +| \vert | | notmuch-show-pipe-message | notmuch-show-pipe-message | +| [remap undo] | notmuch-tag-undo | notmuch-tag-undo | notmuch-tag-undo | +|--------------+----------------------------------------+-------------------------------------------------------+-----------------------------------------| diff --git a/devel/notmuch-web/nmgunicorn.py b/devel/notmuch-web/nmgunicorn.py new file mode 100644 index 00000000..e71ba12a --- /dev/null +++ b/devel/notmuch-web/nmgunicorn.py @@ -0,0 +1,11 @@ +#!/usr/bin/env python3 + +# to launch nmweb from gunicorn. + +from nmweb import urls, index, search, show +import web + +app = web.application(urls, globals()) + +# get the wsgi app from web.py application object +wsgiapp = app.wsgifunc() diff --git a/devel/notmuch-web/nmweb.py b/devel/notmuch-web/nmweb.py new file mode 100755 index 00000000..928e4863 --- /dev/null +++ b/devel/notmuch-web/nmweb.py @@ -0,0 +1,366 @@ +#!/usr/bin/env python + +from __future__ import absolute_import + +try: + from urllib.parse import quote_plus + from urllib.parse import unquote_plus +except ImportError: + from urllib import quote_plus + from urllib import unquote_plus + +from datetime import datetime +from mailbox import MaildirMessage +import mimetypes +import email +import re +import html +import os + +import bleach +import web +from notmuch2 import Database +from jinja2 import Environment, FileSystemLoader # FIXME to PackageLoader +from jinja2 import Markup +try: + import bjoern # from https://github.com/jonashaag/bjoern/ + use_bjoern = True +except: + use_bjoern = False + +# Configuration options +safe_tags = bleach.sanitizer.ALLOWED_TAGS + \ + [u'div', u'span', u'p', u'br', u'table', u'tr', u'td', u'th'] +linkify_plaintext = True # delays page load by about 0.02s of 0.20s budget +show_thread_nav = True # delays page load by about 0.04s of 0.20s budget + +prefix = os.environ.get('NMWEB_PREFIX', "http://localhost:8080") +webprefix = os.environ.get('NMWEB_STATIC', prefix + "/static") +cachedir = os.environ.get('NMWEB_CACHE', "static/cache") # special for webpy server; changeable if using your own +cachepath = os.environ.get('NMWEB_CACHE_PATH', cachedir) # location of static cache in the local filesystem + +if 'NMWEB_DEBUG' in os.environ: + web.config.debug = True +else: + web.config.debug = False + +# End of config options + +env = Environment(autoescape=True, + loader=FileSystemLoader('templates')) + +urls = ( + '/', 'index', + '/search/(.*)', 'search', + '/show/(.*)', 'show', +) + +def urlencode_filter(s): + if type(s) == 'Markup': + s = s.unescape() + s = s.encode('utf8') + s = quote_plus(s) + return Markup(s) +env.filters['url'] = urlencode_filter + +class index: + def GET(self): + web.header('Content-type', 'text/html') + base = env.get_template('base.html') + template = env.get_template('index.html') + db = Database() + tags = db.tags + return template.render(tags=tags, + title="Notmuch webmail", + prefix=prefix, + sprefix=webprefix) + +class search: + def GET(self, terms): + redir = False + if web.input(terms=None).terms: + redir = True + terms = web.input().terms + terms = unquote_plus (terms) + if web.input(afters=None).afters: + afters = web.input(afters=None).afters[:-3] + else: + afters = '0' + if web.input(befores=None).befores: + befores = web.input(befores=None).befores + else: + befores = '4294967296' # 2^32 + try: + if int(afters) > 0 or int(befores) < 4294967296: + redir = True + terms += ' date:@%s..@%s' % (int(afters), int(befores)) + except ValueError: + pass + if redir: + raise web.seeother('/search/%s' % quote_plus(terms.encode('utf8'))) + web.header('Content-type', 'text/html') + db = Database() + ts = db.threads(query=terms, sort=Database.SORT.NEWEST_FIRST) + template = env.get_template('search.html') + return template.generate(terms=terms, + ts=ts, + title=terms, + prefix=prefix, + sprefix=webprefix) + +def format_time_range(start, end): + if end-start < (60*60*24): + time = datetime.fromtimestamp(start).strftime('%Y %b %d %H:%M') + else: + start = datetime.fromtimestamp(start).strftime("%Y %b %d") + end = datetime.fromtimestamp(end).strftime("%Y %b %d") + time = "%s through %s" % (start, end) + return time +env.globals['format_time_range'] = format_time_range + +def mailto_addrs(msg,header_name): + try: + hdr = msg.header(header_name) + except LookupError: + return '' + + frm = email.utils.getaddresses([hdr]) + return ','.join(['%s ' % ((l, p) if p else (l, l)) for (p, l) in frm]) +env.globals['mailto_addrs'] = mailto_addrs + +def link_msg(msg): + lnk = quote_plus(msg.messageid.encode('utf8')) + try: + subj = msg.header('Subject') + except LookupError: + subj = "" + out = '%s' % (prefix, lnk, subj) + return out +env.globals['link_msg'] = link_msg + +def show_msgs(msgs): + r = '
    ' + for msg in msgs: + red = 'color:black; font-style:normal' + if msg.matched: + red = 'color:red; font-style:italic' + frm = mailto_addrs(msg,'From') + lnk = link_msg(msg) + tags = ", ".join(msg.tags) + rs = show_msgs(msg.replies()) + r += '
  • %s—%s [%s] %s
    • ' % (red, frm, lnk, tags, rs) + r += '
' + return r +env.globals['show_msgs'] = show_msgs + +# As email.message.walk, but showing close tags as well +def mywalk(self): + yield self + if self.is_multipart(): + for subpart in self.get_payload(): + for subsubpart in mywalk(subpart): + yield subsubpart + yield 'close-div' + +class show: + def GET(self, mid): + web.header('Content-type', 'text/html') + db = Database() + try: + m = db.find(mid) + except: + raise web.notfound("No such message id.") + template = env.get_template('show.html') + # FIXME add reply-all link with email.urils.getaddresses + # FIXME add forward link using mailto with body parameter? + return template.render(m=m, + mid=mid, + title=m.header('Subject'), + prefix=prefix, + sprefix=webprefix) + +def thread_nav(m): + if not show_thread_nav: return + db = Database() + thread = next(db.threads('thread:'+m.threadid)) + prv = None + found = False + nxt = None + for msg in thread: + if m == msg: + found = True + elif not found: + prv = msg + else: # found message, but not on this loop + nxt = msg + break + yield "
    " + if prv: yield "
  • Previous message (by thread): %s
    • " % link_msg(prv) + if nxt: yield "
  • Next message (by thread): %s
    • " % link_msg(nxt) + yield "

Thread:

" + # FIXME show now takes three queries instead of 1; + # can we yield the message body while computing the thread shape? + thread = next(db.threads('thread:'+m.threadid)) + yield show_msgs(thread.toplevel()) + return +env.globals['thread_nav'] = thread_nav + +def format_message(nm_msg, mid): + fn = list(nm_msg.filenames())[0] + msg = MaildirMessage(open(fn)) + return format_message_walk(msg, mid) + +def decodeAnyway(txt, charset='ascii'): + try: + out = txt.decode(charset) + except: + try: + out = txt.decode('utf-8') + except UnicodeDecodeError: + out = txt.decode('latin1') + return out + +def require_protocol_prefix(attrs, new=False): + if not new: + return attrs + link_text = attrs[u'_text'] + if link_text.startswith(('http:', 'https:', 'mailto:', 'git:', 'id:')): + return attrs + return None + +# Bleach doesn't even try to linkify id:... text, so no point invoking this yet +def modify_id_links(attrs, new=False): + if attrs[(None, u'href')].startswith(u'id:'): + attrs[(None, u'href')] = prefix + "/show/" + attrs[(None, u'href')][3:] + return attrs + +def css_part_id(content_type, parts=[]): + c = content_type.replace('/', '-') + out = "-".join(parts + [c]) + return out + +def format_message_walk(msg, mid): + counter = 0 + cid_refd = [] + parts = ['main'] + for part in mywalk(msg): + if part == 'close-div': + parts.pop() + yield '' + elif part.get_content_maintype() == 'multipart': + yield '
' % \ + (part.get_content_subtype(), css_part_id(part.get_content_type(), parts)) + parts.append(part.get_content_subtype()) + if part.get_content_subtype() == 'alternative': + yield '
    ' + for subpart in part.get_payload(): + yield ('
  • %s
    • ' % + (css_part_id(subpart.get_content_type(), parts), + subpart.get_content_type())) + yield '
' + elif part.get_content_type() == 'message/rfc822': + # FIXME extract subject, date, to/cc/from into a separate template and use it here + yield '
' + elif part.get_content_maintype() == 'text': + if part.get_content_subtype() == 'plain': + yield '
' % css_part_id(part.get_content_type(), parts) + yield '
'
+        out = part.get_payload(decode=True)
+        out = decodeAnyway(out, part.get_content_charset('ascii'))
+        out = html.escape(out)
+        out = out.encode('ascii', 'xmlcharrefreplace').decode('ascii')
+        if linkify_plaintext: out = bleach.linkify(out, callbacks=[require_protocol_prefix])
+        yield out
+        yield '
' + elif part.get_content_subtype() == 'html': + yield '
' % css_part_id(part.get_content_type(), parts) + unb64 = part.get_payload(decode=True) + decoded = decodeAnyway(unb64, part.get_content_charset('ascii')) + cid_refd += find_cids(decoded) + part.set_payload(bleach.clean(replace_cids(decoded, mid), tags=safe_tags). + encode(part.get_content_charset('ascii'), 'xmlcharrefreplace')) + (filename, cid) = link_to_cached_file(part, mid, counter) + counter += 1 + yield '' % \ + os.path.join(prefix, cachedir, mid, filename) + yield '
' + else: + yield '
' % css_part_id(part.get_content_type(), parts) + (filename, cid) = link_to_cached_file(part, mid, counter) + counter += 1 + yield '%s (%s)' % (os.path.join(prefix, + cachedir, + mid, + filename), + filename, + part.get_content_type()) + yield '
' + elif part.get_content_maintype() == 'image': + (filename, cid) = link_to_cached_file(part, mid, counter) + if cid not in cid_refd: + counter += 1 + yield '%s' % (os.path.join(prefix, + cachedir, + mid, + filename), + filename) + else: + (filename, cid) = link_to_cached_file(part, mid, counter) + counter += 1 + yield '%s (%s)' % (os.path.join(prefix, + cachedir, + mid, + filename), + filename, + part.get_content_type()) +env.globals['format_message'] = format_message + +def replace_cids(body, mid): + return body.replace('cid:', os.path.join(prefix, cachedir, mid)+'/') + +def find_cids(body): + return re.findall(r'cid:([^ "\'>]*)', body) + +def link_to_cached_file(part, mid, counter): + filename = part.get_filename() + if not filename: + ext = mimetypes.guess_extension(part.get_content_type()) + if not ext: + ext = '.bin' + filename = 'part-%03d%s' % (counter, ext) + try: + os.makedirs(os.path.join(cachepath, mid)) + except OSError: + pass + fn = os.path.join(cachepath, mid, filename) # FIXME escape mid, filename + fp = open(fn, 'wb') + if part.get_content_maintype() == 'text': + data = part.get_payload(decode=True) + data = decodeAnyway(data, part.get_content_charset('ascii')).encode('utf-8') + else: + try: + data = part.get_payload(decode=True) + except: + data = part.get_payload(decode=False) + if data: + fp.write(data) + fp.close() + if 'Content-ID' in part: + cid = part['Content-ID'] + if cid[0] == '<' and cid[-1] == '>': cid = cid[1:-1] + cid_fn = os.path.join(cachepath, mid, cid) # FIXME escape mid, cid + try: + os.unlink(cid_fn) + except OSError: + pass + os.link(fn, cid_fn) + return (filename, cid) + else: + return (filename, None) + +if __name__ == '__main__': + app = web.application(urls, globals()) + if use_bjoern: + bjoern.run(app.wsgifunc(), "127.0.0.1", 8080) + else: + app.run() diff --git a/devel/notmuch-web/static/css/jquery-ui.css b/devel/notmuch-web/static/css/jquery-ui.css new file mode 120000 index 00000000..eba7c769 --- /dev/null +++ b/devel/notmuch-web/static/css/jquery-ui.css @@ -0,0 +1 @@ +/usr/share/javascript/jquery-ui/themes/base/jquery-ui.min.css \ No newline at end of file diff --git a/devel/notmuch-web/static/css/notmuch-0.1.css b/devel/notmuch-web/static/css/notmuch-0.1.css new file mode 100644 index 00000000..0f085644 --- /dev/null +++ b/devel/notmuch-web/static/css/notmuch-0.1.css @@ -0,0 +1,15 @@ +pre { + white-space: pre-wrap; +} + +.message-rfc822 { + border: 1px solid; + border-radius: 25px; +} + +.embedded-html { + frameborder: 0; + border: 0; + scrolling: no; + width: 100%; +} diff --git a/devel/notmuch-web/static/js/jquery-ui.js b/devel/notmuch-web/static/js/jquery-ui.js new file mode 120000 index 00000000..5c053bab --- /dev/null +++ b/devel/notmuch-web/static/js/jquery-ui.js @@ -0,0 +1 @@ +/usr/share/javascript/jquery-ui/jquery-ui.min.js \ No newline at end of file diff --git a/devel/notmuch-web/static/js/jquery.js b/devel/notmuch-web/static/js/jquery.js new file mode 120000 index 00000000..7fff8870 --- /dev/null +++ b/devel/notmuch-web/static/js/jquery.js @@ -0,0 +1 @@ +/usr/share/javascript/jquery/jquery.min.js \ No newline at end of file diff --git a/devel/notmuch-web/static/js/notmuch-0.1.js b/devel/notmuch-web/static/js/notmuch-0.1.js new file mode 100644 index 00000000..ed6e9f4a --- /dev/null +++ b/devel/notmuch-web/static/js/notmuch-0.1.js @@ -0,0 +1,35 @@ +$(function(){ + $("#after").datepicker({ + altField: "#afters", + altFormat: "@", + changeMonth: true, + changeYear: true, + defaultDate: "-7d", + minDate: "01/01/1970", + yearRange: "2000:+0", + onSelect: function(selectedDate) { + $("#before").datepicker("option","minDate",selectedDate); + } + }); + $("#before").datepicker({ + altField: "#befores", + altFormat: "@", + changeMonth: true, + changeYear: true, + defaultDate: "+1d", + maxDate: "+1d", + yearRange: "2000:+0", + onSelect: function(selectedDate) { + $("#after").datepicker("option","maxDate",selectedDate); + } + }); + $(function(){ + $('.multipart-alternative').tabs() + }); + $(function(){ + $('.embedded-html').on('load',function(){ + this.style.height = this.contentWindow.document.body.offsetHeight + 'px'; + }); + }); +}); + diff --git a/devel/notmuch-web/templates/base.html b/devel/notmuch-web/templates/base.html new file mode 100644 index 00000000..90d92931 --- /dev/null +++ b/devel/notmuch-web/templates/base.html @@ -0,0 +1,39 @@ + + + + + + + + + + +{{title}} + +
+
+{% block searchform %} +
+ + + + +
+{% endblock searchform %} +

{{title}}

+
+
+{% block content %} +

Common tags

+ +
+{% endblock content %} +
+ diff --git a/devel/notmuch-web/templates/index.html b/devel/notmuch-web/templates/index.html new file mode 100644 index 00000000..0eb3fd3c --- /dev/null +++ b/devel/notmuch-web/templates/index.html @@ -0,0 +1,9 @@ +{% extends "base.html" %} +{% block content %} +

Common tags

+ +{% endblock content %} diff --git a/devel/notmuch-web/templates/search.html b/devel/notmuch-web/templates/search.html new file mode 100644 index 00000000..6719c356 --- /dev/null +++ b/devel/notmuch-web/templates/search.html @@ -0,0 +1,10 @@ +{% extends "base.html" %} +

{{ terms|e }}

+{% block content %} +{% for t in ts %} +

{{ t.subject|e }}

+

{{ t.authors|e }}

+

{{ format_time_range(t.first,t.last)|e }}

+ {{ show_msgs(t.toplevel())|safe }} +{% endfor %} +{% endblock content %} diff --git a/devel/notmuch-web/templates/show.html b/devel/notmuch-web/templates/show.html new file mode 100644 index 00000000..98d36acc --- /dev/null +++ b/devel/notmuch-web/templates/show.html @@ -0,0 +1,15 @@ +{% extends "base.html" %} +{% block content %} +{% set headers = ['Subject', 'Date'] %} +{% set addr_headers = ['To', 'Cc', 'From'] %} +{% for header in headers: %} +

{{header}}:{{m.header(header)|e}}

+{% endfor %} +{% for header in addr_headers: %} +

{{header}}:{{mailto_addrs(m,header)|safe}}

+{% endfor %} +
+{% for part in format_message(m,mid): %}{{ part|safe }}{% endfor %} +{% for b in thread_nav(m): %}{{b|safe}}{% endfor %} +
+{% endblock content %} diff --git a/devel/notmuch-web/todo b/devel/notmuch-web/todo new file mode 100644 index 00000000..3c885bd9 --- /dev/null +++ b/devel/notmuch-web/todo @@ -0,0 +1,14 @@ +review escaping and safety handling mail from Bad People + +revise template loader---can we make this faster? + +add reply-all link with email.urils.getaddresses + +change current reply links to quote body + +add forward link using mailto with body parameter? + +unescape the current search term, including translating back dates + + +later: json support, iOS app? diff --git a/devel/release-checks.sh b/devel/release-checks.sh index 23c29eaa..c0accf78 100755 --- a/devel/release-checks.sh +++ b/devel/release-checks.sh @@ -59,7 +59,7 @@ readonly VERSION # In the rest of this file, tests collect list of errors to be fixed -echo -n "Checking that git working directory is clean... " +printf %s "Checking that git working directory is clean... " git_status=`git status --porcelain` if [ "$git_status" = '' ] then @@ -77,7 +77,7 @@ verfail () append_emsg " Please follow the instructions in RELEASING to choose a version" } -echo -n "Checking that '$VERSION' is good with digits and periods... " +printf %s "Checking that '$VERSION' is good with digits and periods... " case $VERSION in *[!0-9.]*) verfail "'$VERSION' contains other characters than digits and periods" ;; @@ -88,7 +88,7 @@ case $VERSION in *) verfail "'$VERSION' is a single number" ;; esac -echo -n "Checking that this is Debian package for notmuch... " +printf %s "Checking that this is Debian package for notmuch... " read deb_notmuch deb_version rest < debian/changelog if [ "$deb_notmuch" = 'notmuch' ] then @@ -98,7 +98,7 @@ else append_emsg "Package name '$deb_notmuch' is not 'notmuch' in debian/changelog" fi -echo -n "Checking that Debian package version is $VERSION-1... " +printf %s "Checking that Debian package version is $VERSION-1... " if [ "$deb_version" = "($VERSION-1)" ] then @@ -108,7 +108,7 @@ else append_emsg "Version '$deb_version' is not '($VERSION-1)' in debian/changelog" fi -echo -n "Checking that python bindings version is $VERSION... " +printf %s "Checking that python bindings version is $VERSION... " py_version=`python3 -c "with open('$PV_FILE') as vf: exec(vf.read()); print(__VERSION__)"` if [ "$py_version" = "$VERSION" ] then @@ -118,7 +118,7 @@ else append_emsg "Version '$py_version' is not '$VERSION' in $PV_FILE" fi -echo -n "Checking that NEWS header is tidy... " +printf %s "Checking that NEWS header is tidy... " if [ "`exec sed 's/./=/g; 1q' NEWS`" = "`exec sed '1d; 2q' NEWS`" ] then echo Yes. @@ -132,7 +132,7 @@ else fi fi -echo -n "Checking that this is Notmuch NEWS... " +printf %s "Checking that this is Notmuch NEWS... " read news_notmuch news_version news_date < NEWS if [ "$news_notmuch" = "Notmuch" ] then @@ -142,7 +142,7 @@ else append_emsg "First word '$news_notmuch' is not 'Notmuch' in NEWS file" fi -echo -n "Checking that NEWS version is $VERSION... " +printf %s "Checking that NEWS version is $VERSION... " if [ "$news_version" = "$VERSION" ] then echo Yes. @@ -154,7 +154,7 @@ fi #eval `date '+year=%Y mon=%m day=%d'` today0utc=`date --date=0Z +%s` # gnu date feature -echo -n "Checking that NEWS date is right... " +printf %s "Checking that NEWS date is right... " case $news_date in '('[2-9][0-9][0-9][0-9]-[01][0-9]-[0123][0-9]')') newsdate0utc=`nd=${news_date#\\(}; date --date="${nd%)} 0Z" +%s` @@ -176,7 +176,7 @@ case $news_date in esac year=`exec date +%Y` -echo -n "Checking that copyright in documentation contains 2009-$year... " +printf %s "Checking that copyright in documentation contains 2009-$year... " # Read the value of variable `copyright' defined in 'doc/conf.py'. copyrightline=$(grep ^copyright doc/conf.py) case $copyrightline in diff --git a/doc/man1/notmuch-config.rst b/doc/man1/notmuch-config.rst index 41e1338b..acc5ecec 100644 --- a/doc/man1/notmuch-config.rst +++ b/doc/man1/notmuch-config.rst @@ -55,22 +55,19 @@ The available configuration items are described below. Non-absolute paths are presumed relative to `$HOME` for items in section **database**. -database.path - Notmuch will store its database here, (in - sub-directory named ``.notmuch`` if **database.mail\_root** - is unset). - - Default: see :ref:`database` +built_with. + Compile time feature . Current possibilities include + "retry_lock" (configure option, included by default). + (since notmuch 0.30, "compact" and "field_processor" are + always included.) -database.mail_root - The top-level directory where your mail currently exists and to - where mail will be delivered in the future. Files should be - individual email messages. +database.autocommit - History: this configuration value was introduced in notmuch 0.32. + How often to commit transactions to disk. `0` means wait until + command completes, otherwise an integer `n` specifies to commit to + disk after every `n` completed transactions. - Default: For compatibility with older configurations, the value of - database.path is used if **database.mail\_root** is unset. + History: this configuration value was introduced in notmuch 0.33. database.backup_dir Directory to store tag dumps when upgrading database. @@ -88,109 +85,26 @@ database.hook_dir Default: See HOOKS, below. -database.autocommit - - How often to commit transactions to disk. `0` means wait until - command completes, otherwise an integer `n` specifies to commit to - disk after every `n` completed transactions. - - History: this configuration value was introduced in notmuch 0.33. - -user.name - Your full name. - - Default: ``$NAME`` variable if set, otherwise read from - ``/etc/passwd``. - -user.primary\_email - Your primary email address. - - Default: ``$EMAIL`` variable if set, otherwise constructed from - the username and hostname of the current machine. - -user.other\_email - A list of other email addresses at which you receive email. - - Default: not set. - -new.tags - A list of tags that will be added to all messages incorporated by - **notmuch new**. - - Default: ``unread;inbox``. - -new.ignore - A list to specify files and directories that will not be searched - for messages by :any:`notmuch-new(1)`. Each entry in the list is either: - - A file or a directory name, without path, that will be ignored, - regardless of the location in the mail store directory hierarchy. - - Or: - - A regular expression delimited with // that will be matched - against the path of the file or directory relative to the database - path. Matching files and directories will be ignored. The - beginning and end of string must be explicitly anchored. For - example, /.*/foo$/ would match "bar/foo" and "bar/baz/foo", but - not "foo" or "bar/foobar". - - Default: empty list. - -search.exclude\_tags - A list of tags that will be excluded from search results by - default. Using an excluded tag in a query will override that - exclusion. - - Default: empty list. Note that :any:`notmuch-setup(1)` puts - ``deleted;spam`` here when creating new configuration file. - -.. _show.extra_headers: - -show.extra\_headers +.. _database.mail_root: - By default :any:`notmuch-show(1)` includes the following headers - in structured output if they are present in the message: - `Subject`, `From`, `To`, `Cc`, `Bcc`, `Reply-To`, `Date`. This - option allows the specification of a list of further - headers to output. - - History: This configuration value was introduced in notmuch 0.35. - - Default: empty list. +database.mail_root + The top-level directory where your mail currently exists and to + where mail will be delivered in the future. Files should be + individual email messages. -maildir.synchronize\_flags - If true, then the following maildir flags (in message filenames) - will be synchronized with the corresponding notmuch tags: + History: this configuration value was introduced in notmuch 0.32. - +--------+-----------------------------------------------+ - | Flag | Tag | - +========+===============================================+ - | D | draft | - +--------+-----------------------------------------------+ - | F | flagged | - +--------+-----------------------------------------------+ - | P | passed | - +--------+-----------------------------------------------+ - | R | replied | - +--------+-----------------------------------------------+ - | S | unread (added when 'S' flag is not present) | - +--------+-----------------------------------------------+ + Default: For compatibility with older configurations, the value of + database.path is used if **database.mail\_root** is unset. - The :any:`notmuch-new(1)` command will notice flag changes in - filenames and update tags, while the :any:`notmuch-tag(1)` and - :any:`notmuch-restore(1)` commands will notice tag changes and - update flags in filenames. +database.path + Notmuch will store its database here, (in + sub-directory named ``.notmuch`` if **database.mail\_root** + is unset). - If there have been any changes in the maildir (new messages added, - old ones removed or renamed, maildir flags changed, etc.), it is - advisable to run :any:`notmuch-new(1)` before - :any:`notmuch-tag(1)` or :any:`notmuch-restore(1)` commands to - ensure the tag changes are properly synchronized to the maildir - flags, as the commands expect the database and maildir to be in - sync. + Default: see :ref:`database` - Default: ``true``. +.. _index.decrypt: index.decrypt Policy for decrypting encrypted messages during indexing. Must be @@ -245,6 +159,8 @@ index.decrypt Default: ``auto``. +.. _index.header: + index.header. Define the query prefix , based on a mail header. For example ``index.header.List=List-Id`` will add a probabilistic @@ -254,22 +170,121 @@ index.header. supported. See :any:`notmuch-search-terms(7)` for a list of existing prefixes, and an explanation of probabilistic prefixes. -built_with. - Compile time feature . Current possibilities include - "retry_lock" (configure option, included by default). - (since notmuch 0.30, "compact" and "field_processor" are - always included.) +.. _maildir.synchronize_flags: + +maildir.synchronize\_flags + If true, then the following maildir flags (in message filenames) + will be synchronized with the corresponding notmuch tags: + + +--------+-----------------------------------------------+ + | Flag | Tag | + +========+===============================================+ + | D | draft | + +--------+-----------------------------------------------+ + | F | flagged | + +--------+-----------------------------------------------+ + | P | passed | + +--------+-----------------------------------------------+ + | R | replied | + +--------+-----------------------------------------------+ + | S | unread (added when 'S' flag is not present) | + +--------+-----------------------------------------------+ + + The :any:`notmuch-new(1)` command will notice flag changes in + filenames and update tags, while the :any:`notmuch-tag(1)` and + :any:`notmuch-restore(1)` commands will notice tag changes and + update flags in filenames. + + If there have been any changes in the maildir (new messages added, + old ones removed or renamed, maildir flags changed, etc.), it is + advisable to run :any:`notmuch-new(1)` before + :any:`notmuch-tag(1)` or :any:`notmuch-restore(1)` commands to + ensure the tag changes are properly synchronized to the maildir + flags, as the commands expect the database and maildir to be in + sync. + + Default: ``true``. + +.. _new.ignore: + +new.ignore + A list to specify files and directories that will not be searched + for messages by :any:`notmuch-new(1)`. Each entry in the list is either: + + A file or a directory name, without path, that will be ignored, + regardless of the location in the mail store directory hierarchy. + + Or: + + A regular expression delimited with // that will be matched + against the path of the file or directory relative to the database + path. Matching files and directories will be ignored. The + beginning and end of string must be explicitly anchored. For + example, /.*/foo$/ would match "bar/foo" and "bar/baz/foo", but + not "foo" or "bar/foobar". + + Default: empty list. + +.. _new.tags: + +new.tags + A list of tags that will be added to all messages incorporated by + **notmuch new**. + + Default: ``unread;inbox``. query. Expansion for named query called . See :any:`notmuch-search-terms(7)` for more information about named queries. +search.exclude\_tags + A list of tags that will be excluded from search results by + default. Using an excluded tag in a query will override that + exclusion. + + Default: empty list. Note that :any:`notmuch-setup(1)` puts + ``deleted;spam`` here when creating new configuration file. + +.. _show.extra_headers: + +show.extra\_headers + + By default :any:`notmuch-show(1)` includes the following headers + in structured output if they are present in the message: + `Subject`, `From`, `To`, `Cc`, `Bcc`, `Reply-To`, `Date`. This + option allows the specification of a list of further + headers to output. + + History: This configuration value was introduced in notmuch 0.35. + + Default: empty list. + squery. Expansion for named query called , using s-expression syntax. See :any:`notmuch-sexp-queries(7)` for more information about s-expression queries. +user.name + Your full name. + + Default: ``$NAME`` variable if set, otherwise read from + ``/etc/passwd``. + +user.other\_email + A list of other email addresses at which you receive email + (see also, :ref:`user.primary_email `). + + Default: not set. + +.. _user.primary_email: + +user.primary\_email + Your primary email address. + + Default: ``$EMAIL`` variable if set, otherwise constructed from + the username and hostname of the current machine. + FILES ===== diff --git a/doc/man1/notmuch-insert.rst b/doc/man1/notmuch-insert.rst index da9ca791..fe2bf26b 100644 --- a/doc/man1/notmuch-insert.rst +++ b/doc/man1/notmuch-insert.rst @@ -14,12 +14,12 @@ DESCRIPTION **notmuch insert** reads a message from standard input and delivers it into the maildir directory given by configuration option -**database.mail_root**, then incorporates the message into the notmuch +:ref:`database.mail_root `, then incorporates the message into the notmuch database. It is an alternative to using a separate tool to deliver the message then running :any:`notmuch-new(1)` afterwards. The new message will be tagged with the tags specified by the -**new.tags** configuration option, then by operations specified on the +:ref:`new.tags ` configuration option, then by operations specified on the command-line: tags prefixed by '+' are added while those prefixed by '-' are removed. @@ -86,7 +86,17 @@ Supported options for **insert** include ``--decrypt=nostash`` without considering the security of your index. - See also ``index.decrypt`` in :any:`notmuch-config(1)`. + See also :ref:`index.decrypt ` in :any:`notmuch-config(1)`. + +CONFIGURATION +============= + +Indexing is influenced by the configuration options +:ref:`index.decrypt ` and :ref:`index.header +`. Tagging +is controlled by :ref:`new.tags ` and +:ref:`maildir.synchronize_flags `. See +:any:`notmuch-config(1)` for details. EXIT STATUS =========== diff --git a/doc/man1/notmuch-new.rst b/doc/man1/notmuch-new.rst index 9cb4a54e..398f8813 100644 --- a/doc/man1/notmuch-new.rst +++ b/doc/man1/notmuch-new.rst @@ -78,6 +78,16 @@ Supported options for **new** include to optimize the scanning of directories for new mail. This option turns that optimization off. +CONFIGURATION +============= + +Indexing is influenced by the configuration options +:ref:`index.decrypt `, :ref:`index.header +`, and :ref:`new.ignore `. Tagging +is controlled by :ref:`new.tags ` and +:ref:`maildir.synchronize_flags `. See +:any:`notmuch-config(1)` for details. + EXIT STATUS =========== diff --git a/doc/man5/notmuch-hooks.rst b/doc/man5/notmuch-hooks.rst index 0ab5efbc..d778bdb8 100644 --- a/doc/man5/notmuch-hooks.rst +++ b/doc/man5/notmuch-hooks.rst @@ -30,9 +30,9 @@ pre-new post-new This hook is invoked by the :any:`notmuch-new(1)` command after - new messages have been imported into the database and initial tags - have been applied. The hook will not be run if there have been any - errors during the scan or import. + any new messages have been imported into the database and initial + tags have been applied. The hook will not be run if there have + been any errors during the scan or import. Typically this hook is used to perform additional query-based tagging on the imported messages. diff --git a/doc/man7/notmuch-search-terms.rst b/doc/man7/notmuch-search-terms.rst index e80cc7d0..4f616b7e 100644 --- a/doc/man7/notmuch-search-terms.rst +++ b/doc/man7/notmuch-search-terms.rst @@ -169,6 +169,12 @@ property:= can be present on a given message with several different values. See :any:`notmuch-properties(7)` for more details. +sexp: + The **sexp:** prefix allows subqueries in the format + documented in :any:`notmuch-sexp-queries(7)`. Note that subqueries containing + spaces must be quoted, and any embedded double quotes must be escaped + (see :any:`quoting`). + User defined prefixes are also supported, see :any:`notmuch-config(1)` for details. @@ -257,7 +263,7 @@ Boolean Probabilistic **body:**, **to:**, **attachment:**, **mimetype:** Special - **from:**, **query:**, **subject:** + **from:**, **query:**, **subject:**, **sexp:** Terms and phrases ----------------- @@ -275,11 +281,13 @@ the same phrase. - a.list.of.words Both parenthesised lists of terms and quoted phrases are ok with -probabilistic prefixes such as **to:**, **from:**, and **subject:**. In particular +probabilistic prefixes such as **to:**, **from:**, and **subject:**. +For prefixes supporting regex search, the parenthesised list should be +quoted. In particular :: - subject:(pizza free) + subject:"(pizza free)" is equivalent to @@ -295,6 +303,8 @@ Both of these will match a subject "Free Delicious Pizza" while will not. +.. _quoting: + Quoting ------- @@ -322,6 +332,13 @@ e.g. % notmuch search 'folder:"/^.*/(Junk|Spam)$/"' % notmuch search 'thread:"{from:mallory and date:2009}" and thread:{to:mallory}' +Double quotes within query strings need to be doubled to escape them. + +:: + + % notmuch search 'tag:"""quoted tag"""' + % notmuch search 'sexp:"(or ""wizard"" ""php"")"' + DATE AND TIME SEARCH ==================== diff --git a/doc/notmuch-emacs.rst b/doc/notmuch-emacs.rst index 85b2c0ea..41f62390 100644 --- a/doc/notmuch-emacs.rst +++ b/doc/notmuch-emacs.rst @@ -331,11 +331,21 @@ tags. As is the case with :ref:`notmuch-search`, the presentation of results can be controlled by the variable ``notmuch-search-oldest-first``. +.. _notmuch-unthreaded: + +notmuch-unthreaded +------------------ + +``notmuch-unthreaded-mode`` is similar to :any:`notmuch-tree` in that +each line corresponds to a single message, but no thread information +is presented. + +Keybindings are the same as :any:`notmuch-tree`. Global key bindings =================== -Several features are accessible from anywhere in notmuch through the +Several features are accessible from most places in notmuch through the following key bindings: ``j`` @@ -344,6 +354,8 @@ following key bindings: ``k`` Tagging operations using :ref:`notmuch-tag-jump` +``C-_`` ``C-/`` ``C-x u``: Undo previous tagging operation using :ref:`notmuch-tag-undo` + .. _notmuch-jump: notmuch-jump @@ -373,6 +385,21 @@ operations specified in ``notmuch-tagging-keys``; i.e. each |docstring::notmuch-tagging-keys| +.. _notmuch-tag-undo: + +notmuch-tag-undo +---------------- + +Each notmuch buffer supporting tagging operations (i.e buffers in +:any:`notmuch-show`, :any:`notmuch-search`, :any:`notmuch-tree`, and +:any:`notmuch-unthreaded` mode) keeps a local stack of tagging +operations. These can be undone via ``notmuch-tag-undo``. By default +this is bound to the usual Emacs keys for undo. + +:index:`notmuch-tag-undo` + + |docstring::notmuch-tag-undo| + Buffer navigation ================= diff --git a/emacs/notmuch-hello.el b/emacs/notmuch-hello.el index beb25382..4662e704 100644 --- a/emacs/notmuch-hello.el +++ b/emacs/notmuch-hello.el @@ -710,6 +710,9 @@ with `notmuch-hello-query-counts'." ;; that when we modify map it does not modify widget-keymap). (let ((map (make-composed-keymap (list (make-sparse-keymap) widget-keymap)))) (set-keymap-parent map notmuch-common-keymap) + ;; Currently notmuch-hello-mode supports free text entry, but not + ;; tagging operations, so provide standard undo. + (define-key map [remap notmuch-tag-undo] #'undo) map) "Keymap for \"notmuch hello\" buffers.") diff --git a/emacs/notmuch-lib.el b/emacs/notmuch-lib.el index 45817e13..6fc71cc7 100644 --- a/emacs/notmuch-lib.el +++ b/emacs/notmuch-lib.el @@ -166,6 +166,7 @@ For example, if you wanted to remove an \"inbox\" tag and add an (define-key map (kbd "M-=") 'notmuch-refresh-all-buffers) (define-key map "G" 'notmuch-poll-and-refresh-this-buffer) (define-key map "j" 'notmuch-jump-search) + (define-key map [remap undo] 'notmuch-tag-undo) map) "Keymap shared by all notmuch modes.") diff --git a/emacs/notmuch-logo.png b/emacs/notmuch-logo.png deleted file mode 100644 index 53b5e6a4..00000000 Binary files a/emacs/notmuch-logo.png and /dev/null differ diff --git a/emacs/notmuch-tag.el b/emacs/notmuch-tag.el index 8af09e68..95977881 100644 --- a/emacs/notmuch-tag.el +++ b/emacs/notmuch-tag.el @@ -275,6 +275,11 @@ This can be used with `notmuch-tag-format-image-data'." ") +;;; track history of tag operations +(defvar-local notmuch-tag-history nil + "Buffer local history of `notmuch-tag' function.") +(put 'notmuch-tag-history 'permanent-local t) + ;;; Format Handling (defvar notmuch-tag--format-cache (make-hash-table :test 'equal) @@ -458,14 +463,19 @@ from TAGS if present." "Use batch tagging if the tagging query is longer than this. This limits the length of arguments passed to the notmuch CLI to -avoid system argument length limits and performance problems.") +avoid system argument length limits and performance problems. + +NOTE: this variable is no longer used.") + +(make-obsolete-variable 'notmuch-tag-argument-limit nil "notmuch 0.36") -(defun notmuch-tag (query tag-changes) +(defun notmuch-tag (query tag-changes &optional omit-hist) "Add/remove tags in TAG-CHANGES to messages matching QUERY. QUERY should be a string containing the search-terms. -TAG-CHANGES is a list of strings of the form \"+tag\" or -\"-tag\" to add or remove tags, respectively. +TAG-CHANGES is a list of strings of the form \"+tag\" or \"-tag\" +to add or remove tags, respectively. OMIT-HIST disables history +tracking if non-nil. Note: Other code should always use this function to alter tags of messages instead of running (notmuch-call-notmuch-process \"tag\" ..) @@ -481,16 +491,30 @@ notmuch-after-tag-hook will be run." (notmuch-dlet ((tag-changes tag-changes) (query query)) (run-hooks 'notmuch-before-tag-hook)) - (if (<= (length query) notmuch-tag-argument-limit) - (apply 'notmuch-call-notmuch-process "tag" - (append tag-changes (list "--" query))) - ;; Use batch tag mode to avoid argument length limitations - (let ((batch-op (concat (mapconcat #'notmuch-hex-encode tag-changes " ") - " -- " query))) - (notmuch-call-notmuch-process :stdin-string batch-op "tag" "--batch"))) - (notmuch-dlet ((tag-changes tag-changes) - (query query)) - (run-hooks 'notmuch-after-tag-hook)))) + (with-temp-buffer + (insert (concat (mapconcat #'notmuch-hex-encode tag-changes " ") " -- " query)) + (unless (= 0 + (notmuch--call-process-region + (point-min) (point-max) notmuch-command t t nil "tag" "--batch")) + (notmuch-logged-error "notmuch tag failed" (buffer-string)))) + (unless omit-hist + (push (list :query query :tag-changes tag-changes) notmuch-tag-history))) + (notmuch-dlet ((tag-changes tag-changes) + (query query)) + (run-hooks 'notmuch-after-tag-hook))) + +(defun notmuch-tag-undo () + "Undo the previous tagging operation in the current buffer. Uses +buffer local variable `notmuch-tag-history' to determine what +that operation was." + (interactive) + (when (null notmuch-tag-history) + (error "no further notmuch undo information")) + (let* ((action (pop notmuch-tag-history)) + (query (plist-get action :query)) + (changes (notmuch-tag-change-list (plist-get action :tag-changes) t))) + (notmuch-tag query changes t)) + (notmuch-refresh-this-buffer)) (defun notmuch-tag-change-list (tags &optional reverse) "Convert TAGS into a list of tag changes. diff --git a/emacs/notmuch.el b/emacs/notmuch.el index 6abb17ff..c9cf80dc 100644 --- a/emacs/notmuch.el +++ b/emacs/notmuch.el @@ -93,7 +93,7 @@ Supported fields are: date, count, authors, subject, tags. For example: (setq notmuch-search-result-format - '((\"authors\" . \"%-40s\") + \\='((\"authors\" . \"%-40s\") (\"subject\" . \"%s\"))) Line breaks are permitted in format strings (though this is diff --git a/lib/Makefile.local b/lib/Makefile.local index 1378a74b..6d67a2a4 100644 --- a/lib/Makefile.local +++ b/lib/Makefile.local @@ -64,7 +64,8 @@ libnotmuch_cxx_srcs = \ $(dir)/prefix.cc \ $(dir)/open.cc \ $(dir)/init.cc \ - $(dir)/parse-sexp.cc + $(dir)/parse-sexp.cc \ + $(dir)/sexp-fp.cc libnotmuch_modules := $(libnotmuch_c_srcs:.c=.o) $(libnotmuch_cxx_srcs:.cc=.o) diff --git a/lib/database-private.h b/lib/database-private.h index 657b1aa1..419b9fe6 100644 --- a/lib/database-private.h +++ b/lib/database-private.h @@ -354,10 +354,6 @@ _notmuch_query_string_to_xapian_query (notmuch_database_t *notmuch, std::string query_string, Xapian::Query &output, std::string &msg); -/* parse-sexp.cc */ -notmuch_status_t -_notmuch_sexp_string_to_xapian_query (notmuch_database_t *notmuch, const char *querystr, - Xapian::Query &output); notmuch_status_t _notmuch_query_expand (notmuch_database_t *notmuch, const char *field, Xapian::Query subquery, diff --git a/lib/prefix.cc b/lib/prefix.cc index 857c05b9..06e2333a 100644 --- a/lib/prefix.cc +++ b/lib/prefix.cc @@ -3,6 +3,7 @@ #include "thread-fp.h" #include "regexp-fields.h" #include "parse-time-vrp.h" +#include "sexp-fp.h" typedef struct { const char *name; @@ -60,6 +61,8 @@ prefix_t prefix_table[] = { NOTMUCH_FIELD_PROCESSOR }, { "query", NULL, NOTMUCH_FIELD_EXTERNAL | NOTMUCH_FIELD_PROCESSOR }, + { "sexp", NULL, NOTMUCH_FIELD_EXTERNAL | + NOTMUCH_FIELD_PROCESSOR }, { "from", "XFROM", NOTMUCH_FIELD_EXTERNAL | NOTMUCH_FIELD_PROBABILISTIC | NOTMUCH_FIELD_PROCESSOR }, @@ -138,6 +141,8 @@ _setup_query_field (const prefix_t *prefix, notmuch_database_t *notmuch) fp = (new QueryFieldProcessor (*notmuch->query_parser, notmuch))->release (); else if (STRNCMP_LITERAL (prefix->name, "thread") == 0) fp = (new ThreadFieldProcessor (*notmuch->query_parser, notmuch))->release (); + else if (STRNCMP_LITERAL (prefix->name, "sexp") == 0) + fp = (new SexpFieldProcessor (notmuch))->release (); else fp = (new RegexpFieldProcessor (prefix->name, prefix->flags, *notmuch->query_parser, notmuch))->release (); diff --git a/lib/query.cc b/lib/query.cc index b0937fcc..707f6222 100644 --- a/lib/query.cc +++ b/lib/query.cc @@ -227,6 +227,7 @@ _notmuch_query_ensure_parsed_xapian (notmuch_query_t *query) return NOTMUCH_STATUS_SUCCESS; } +#if HAVE_SFSEXP static notmuch_status_t _notmuch_query_ensure_parsed_sexpr (notmuch_query_t *query) { @@ -243,6 +244,7 @@ _notmuch_query_ensure_parsed_sexpr (notmuch_query_t *query) _notmuch_query_cache_terms (query); return NOTMUCH_STATUS_SUCCESS; } +#endif static notmuch_status_t _notmuch_query_ensure_parsed (notmuch_query_t *query) diff --git a/lib/regexp-fields.cc b/lib/regexp-fields.cc index 7e9d959c..539915d8 100644 --- a/lib/regexp-fields.cc +++ b/lib/regexp-fields.cc @@ -227,7 +227,8 @@ RegexpFieldProcessor::operator() (const std::string & str) * phrase parsing, when possible */ std::string query_str; - if (*str.rbegin () != '*' || str.find (' ') != std::string::npos) + if ((str.at (0) != '(' || *str.rbegin () != ')') && + (*str.rbegin () != '*' || str.find (' ') != std::string::npos)) query_str = '"' + str + '"'; else query_str = str; diff --git a/lib/sexp-fp.cc b/lib/sexp-fp.cc new file mode 100644 index 00000000..1fdf5225 --- /dev/null +++ b/lib/sexp-fp.cc @@ -0,0 +1,44 @@ +/* sexp-fp.cc - "sexp:" field processor glue + * + * This file is part of notmuch. + * + * Copyright © 2022 David Bremner + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see https://www.gnu.org/licenses/ . + * + * Author: David Bremner + */ + +#include "database-private.h" +#include "sexp-fp.h" +#include + +Xapian::Query +SexpFieldProcessor::operator() (const std::string & query_string) +{ + notmuch_status_t status; + Xapian::Query output; + +#if HAVE_SFSEXP + status = _notmuch_sexp_string_to_xapian_query (notmuch, query_string.c_str (), output); + if (status) { + throw Xapian::QueryParserError ("error parsing " + query_string); + } +#else + throw Xapian::QueryParserError ("sexp query parser not available"); +#endif + + return output; + +} diff --git a/lib/sexp-fp.h b/lib/sexp-fp.h new file mode 100644 index 00000000..341dfa7e --- /dev/null +++ b/lib/sexp-fp.h @@ -0,0 +1,41 @@ +/* sexp-fp.h - sexp field processor glue + * + * This file is part of notmuch. + * + * Copyright © 2022 David Bremner + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see https://www.gnu.org/licenses/ . + * + * Author: David Bremner + */ + +#ifndef NOTMUCH_SEXP_FP_H +#define NOTMUCH_SEXP_FP_H + +#include +#include "notmuch.h" + +class SexpFieldProcessor : public Xapian::FieldProcessor { +protected: + notmuch_database_t *notmuch; + +public: + SexpFieldProcessor (notmuch_database_t *notmuch_) : notmuch (notmuch_) + { + }; + + Xapian::Query operator() (const std::string & query_string); +}; + +#endif /* NOTMUCH_SEXP_FP_H */ diff --git a/notmuch-insert.c b/notmuch-insert.c index 214d4d03..e44607ad 100644 --- a/notmuch-insert.c +++ b/notmuch-insert.c @@ -241,6 +241,26 @@ maildir_mktemp (const void *ctx, const char *maildir, bool world_readable, char return fd; } +static bool +write_buf (const char *buf, int fdout, ssize_t remain) +{ + const char *p = buf; + + do { + ssize_t written = write (fdout, p, remain); + if (written < 0 && errno == EINTR) + continue; + if (written <= 0) { + fprintf (stderr, "Error: writing to temporary file: %s", + strerror (errno)); + return false; + } + p += written; + remain -= written; + } while (remain > 0); + return true; +} + /* * Copy fdin to fdout, return true on success, and false on errors and * empty input. @@ -249,11 +269,13 @@ static bool copy_fd (int fdout, int fdin) { bool empty = true; + bool first = true; + const char *header = "X-Envelope-From: "; while (! interrupted) { ssize_t remain; char buf[4096]; - char *p; + const char *p = buf; remain = read (fdin, buf, sizeof (buf)); if (remain == 0) @@ -266,20 +288,18 @@ copy_fd (int fdout, int fdin) return false; } - p = buf; - do { - ssize_t written = write (fdout, p, remain); - if (written < 0 && errno == EINTR) - continue; - if (written <= 0) { - fprintf (stderr, "Error: writing to temporary file: %s", - strerror (errno)); + if (first && remain >= 5 && 0 == strncmp (buf, "From ", 5)) { + if (! write_buf (header, fdout, strlen (header))) return false; - } - p += written; - remain -= written; - empty = false; - } while (remain > 0); + p += 5; + remain -= 5; + } + + first = false; + + if (! write_buf (p, fdout, remain)) + return false; + empty = false; } return (! interrupted && ! empty); diff --git a/performance-test/T06-emacs.sh b/performance-test/T06-emacs.sh new file mode 100755 index 00000000..66f0be58 --- /dev/null +++ b/performance-test/T06-emacs.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash + +test_description='emacs operations' + +. $(dirname "$0")/perf-test-lib.sh || exit 1 +. $NOTMUCH_SRCDIR/test/test-lib-emacs.sh || exit 1 + +test_require_emacs + +time_start + +print_emacs_header + +MSGS=$(notmuch search --output=messages "*" | shuf -n 50 | awk '{printf " \"%s\"",$1}') + +time_emacs "tag messages" \ +"(dolist (msg (list $MSGS)) + (notmuch-tag msg (list \"+test\")) + (notmuch-tag msg (list \"-test\"))))" + +time_done diff --git a/performance-test/perf-test-lib.sh b/performance-test/perf-test-lib.sh index 41b1ddfd..c34f8cd6 100644 --- a/performance-test/perf-test-lib.sh +++ b/performance-test/perf-test-lib.sh @@ -41,6 +41,8 @@ done # Ensure NOTMUCH_SRCDIR and NOTMUCH_BUILDDIR are set. . $(dirname "$0")/../test/export-dirs.sh || exit 1 +. "$NOTMUCH_SRCDIR/test/test-vars.sh" || exit 1 + # Where to run the tests TEST_DIRECTORY=$NOTMUCH_BUILDDIR/performance-test @@ -208,6 +210,11 @@ print_header () printf "\t\t\tWall(s)\tUsr(s)\tSys(s)\tRes(K)\tIn/Out(512B)\n" } +print_emacs_header () +{ + printf "\t\t\tWall(s)\tGCs\tGC time(s)\n" +} + time_run () { printf " %-22s" "$1" diff --git a/test/T055-path-config.sh b/test/T055-path-config.sh index 1df240dd..4897c814 100755 --- a/test/T055-path-config.sh +++ b/test/T055-path-config.sh @@ -293,6 +293,26 @@ user.primary_email=test_suite@notmuchmail.org EOF test_expect_equal_file EXPECTED OUTPUT + test_begin_subtest "Config list from python ($config)" + test_python < OUTPUT +from notmuch2 import Database +db=Database(config=Database.CONFIG.SEARCH) +for key in list(db.config): + print(key) +EOF + cat < EXPECTED +database.autocommit +database.backup_dir +database.hook_dir +database.mail_root +database.path +maildir.synchronize_flags +new.tags +user.name +user.other_email +user.primary_email +EOF + test_expect_equal_file EXPECTED OUTPUT case $config in XDG*) test_begin_subtest "Set shadowed config value in database ($config)" diff --git a/test/T070-insert.sh b/test/T070-insert.sh index ec170b30..e1e3b151 100755 --- a/test/T070-insert.sh +++ b/test/T070-insert.sh @@ -292,4 +292,9 @@ for code in OUT_OF_MEMORY XAPIAN_EXCEPTION ; do test_expect_code 0 "notmuch_with_shim shim-$code insert --keep < \"$gen_msg_filename\"" done +test_begin_subtest "insert converts mboxes on delivery" +notmuch insert +unmboxed < "${TEST_DIRECTORY}"/corpora/indexing/mbox-attachment.eml +output=$(notmuch count tag:unmboxed) +test_expect_equal "${output}" 1 + test_done diff --git a/test/T081-sexpr-search.sh b/test/T081-sexpr-search.sh index e2936cd7..da819190 100755 --- a/test/T081-sexpr-search.sh +++ b/test/T081-sexpr-search.sh @@ -31,6 +31,13 @@ thread:XXX 2009-11-18 [1/3] Carl Worth| Jan Janak; [notmuch] What a great idea EOF test_expect_equal_file EXPECTED OUTPUT +test_begin_subtest "and of stemmed terms" +notmuch search --query=sexp '(and wonderful wizard)' | notmuch_search_sanitize > OUTPUT +cat < EXPECTED +thread:XXX 2009-11-18 [1/3] Carl Worth| Jan Janak; [notmuch] What a great idea! (inbox unread) +EOF +test_expect_equal_file EXPECTED OUTPUT + test_begin_subtest "or of exact terms" notmuch search --query=sexp '(or "php" "wizard")' | notmuch_search_sanitize > OUTPUT cat < EXPECTED @@ -39,6 +46,14 @@ thread:XXX 2009-11-18 [1/3] Carl Worth| Jan Janak; [notmuch] What a great idea EOF test_expect_equal_file EXPECTED OUTPUT +test_begin_subtest "or of exact terms via field processor" +notmuch search 'sexp:"(or ""php"" ""wizard"")"' | notmuch_search_sanitize > OUTPUT +cat < EXPECTED +thread:XXX 2010-12-29 [1/1] François Boulogne; [aur-general] Guidelines: cp, mkdir vs install (inbox unread) +thread:XXX 2009-11-18 [1/3] Carl Worth| Jan Janak; [notmuch] What a great idea! (inbox unread) +EOF +test_expect_equal_file EXPECTED OUTPUT + test_begin_subtest "single term in body" notmuch search --query=sexp 'wizard' | notmuch_search_sanitize>OUTPUT cat < EXPECTED @@ -707,6 +722,11 @@ notmuch search property:foo=bar > EXPECTED notmuch search --query=sexp '(property (rx foo=.*))' > OUTPUT test_expect_equal_file EXPECTED OUTPUT +test_begin_subtest "regexp 'property' search via field processor" +notmuch search property:foo=bar > EXPECTED +notmuch search 'sexp:"(property (rx foo=.*))"' > OUTPUT +test_expect_equal_file EXPECTED OUTPUT + test_begin_subtest "anchored 'tag' search" notmuch search tag:signed > EXPECTED notmuch search --query=sexp '(tag (rx ^si))' > OUTPUT @@ -743,6 +763,13 @@ thread:XXX 2009-11-18 [7/7] Lars Kellogg-Stedman, Mikhail Gusarov, Keith Packa EOF test_expect_equal_file EXPECTED OUTPUT +test_begin_subtest "Compound subquery via field processor" +notmuch search 'sexp:"(thread (of (from keithp) (subject Maildir)))"' | notmuch_search_sanitize > OUTPUT +cat< EXPECTED +thread:XXX 2009-11-18 [7/7] Lars Kellogg-Stedman, Mikhail Gusarov, Keith Packard, Carl Worth; [notmuch] Working with Maildir storage? (inbox signed unread) +EOF +test_expect_equal_file EXPECTED OUTPUT + test_begin_subtest "empty subquery" notmuch search --query=sexp '(thread (of))' 1>OUTPUT 2>&1 notmuch search '*' > EXPECTED @@ -969,6 +996,11 @@ grep -Ril List-Id ${MAIL_DIR} | sort | notmuch_dir_sanitize > EXPECTED notmuch search --output=files --query=sexp '(List *)' | sort | notmuch_dir_sanitize > OUTPUT test_expect_equal_file EXPECTED OUTPUT +test_begin_subtest "wildcard search for user header via field processor" +grep -Ril List-Id ${MAIL_DIR} | sort | notmuch_dir_sanitize > EXPECTED +notmuch search --output=files 'sexp:"(List *)"' | sort | notmuch_dir_sanitize > OUTPUT +test_expect_equal_file EXPECTED OUTPUT + test_begin_subtest "wildcard search for user header 2" grep -Ril List-Id ${MAIL_DIR} | sort | notmuch_dir_sanitize > EXPECTED notmuch search --output=files --query=sexp '(List (starts-with not))' | sort | notmuch_dir_sanitize > OUTPUT diff --git a/test/T090-search-output.sh b/test/T090-search-output.sh index bf28d220..0d85c609 100755 --- a/test/T090-search-output.sh +++ b/test/T090-search-output.sh @@ -435,7 +435,7 @@ test_expect_equal_file EXPECTED OUTPUT test_begin_subtest "search for non-existent message prints nothing" notmuch search "no-message-matches-this" > OUTPUT -echo -n >EXPECTED +: >EXPECTED test_expect_equal_file EXPECTED OUTPUT test_begin_subtest "search --format=json for non-existent message prints proper empty json" diff --git a/test/T190-multipart.sh b/test/T190-multipart.sh index 3545a599..d3b7f87c 100755 --- a/test/T190-multipart.sh +++ b/test/T190-multipart.sh @@ -683,7 +683,7 @@ test_expect_equal_json "$(cat OUTPUT)" "$(cat EXPECTED)" test_begin_subtest "'notmuch show --part' does not corrupt a part with CRLF pair" notmuch show --format=raw --part=3 id:base64-part-with-crlf > crlf.out -echo -n -e "\xEF\x0D\x0A" > crlf.expected +printf "\xEF\x0D\x0A" > crlf.expected test_expect_equal_file crlf.out crlf.expected diff --git a/test/T310-emacs.sh b/test/T310-emacs.sh index a05b828a..9d0df187 100755 --- a/test/T310-emacs.sh +++ b/test/T310-emacs.sh @@ -130,75 +130,6 @@ test_emacs '(notmuch-search "tag:inbox") (test-output)' test_expect_equal_file $EXPECTED/notmuch-show-thread-maildir-storage OUTPUT -test_begin_subtest "Add tag from search view" -os_x_darwin_thread=$(notmuch search --output=threads id:ddd65cda0911171950o4eea4389v86de9525e46052d3@mail.gmail.com) -test_emacs "(notmuch-search \"$os_x_darwin_thread\") - (notmuch-test-wait) - (execute-kbd-macro \"+tag-from-search-view\")" -output=$(notmuch search $os_x_darwin_thread | notmuch_search_sanitize) -test_expect_equal "$output" "thread:XXX 2009-11-18 [4/4] Jjgod Jiang, Alexander Botero-Lowry; [notmuch] Mac OS X/Darwin compatibility issues (inbox tag-from-search-view unread)" - -test_begin_subtest "Remove tag from search view" -test_emacs "(notmuch-search \"$os_x_darwin_thread\") - (notmuch-test-wait) - (execute-kbd-macro \"-tag-from-search-view\")" -output=$(notmuch search $os_x_darwin_thread | notmuch_search_sanitize) -test_expect_equal "$output" "thread:XXX 2009-11-18 [4/4] Jjgod Jiang, Alexander Botero-Lowry; [notmuch] Mac OS X/Darwin compatibility issues (inbox unread)" - -test_begin_subtest "Add tag (large query)" -# We use a long query to force us into batch mode and use a funny tag -# that requires escaping for batch tagging. -test_emacs "(notmuch-tag (concat \"$os_x_darwin_thread\" \" or \" (mapconcat #'identity (make-list notmuch-tag-argument-limit \"x\") \"-\")) (list \"+tag-from-%-large-query\"))" -output=$(notmuch search $os_x_darwin_thread | notmuch_search_sanitize) -test_expect_equal "$output" "thread:XXX 2009-11-18 [4/4] Jjgod Jiang, Alexander Botero-Lowry; [notmuch] Mac OS X/Darwin compatibility issues (inbox tag-from-%-large-query unread)" -notmuch tag -tag-from-%-large-query $os_x_darwin_thread - -test_begin_subtest "notmuch-show: add single tag to single message" -test_emacs "(notmuch-show \"$os_x_darwin_thread\") - (execute-kbd-macro \"+tag-from-show-view\")" -output=$(notmuch search $os_x_darwin_thread | notmuch_search_sanitize) -test_expect_equal "$output" "thread:XXX 2009-11-18 [4/4] Jjgod Jiang, Alexander Botero-Lowry; [notmuch] Mac OS X/Darwin compatibility issues (inbox tag-from-show-view unread)" - -test_begin_subtest "notmuch-show: remove single tag from single message" -test_emacs "(notmuch-show \"$os_x_darwin_thread\") - (execute-kbd-macro \"-tag-from-show-view\")" -output=$(notmuch search $os_x_darwin_thread | notmuch_search_sanitize) -test_expect_equal "$output" "thread:XXX 2009-11-18 [4/4] Jjgod Jiang, Alexander Botero-Lowry; [notmuch] Mac OS X/Darwin compatibility issues (inbox unread)" - -test_begin_subtest "notmuch-show: add multiple tags to single message" -test_emacs "(notmuch-show \"$os_x_darwin_thread\") - (execute-kbd-macro \"+tag1-from-show-view +tag2-from-show-view\")" -output=$(notmuch search $os_x_darwin_thread | notmuch_search_sanitize) -test_expect_equal "$output" "thread:XXX 2009-11-18 [4/4] Jjgod Jiang, Alexander Botero-Lowry; [notmuch] Mac OS X/Darwin compatibility issues (inbox tag1-from-show-view tag2-from-show-view unread)" - -test_begin_subtest "notmuch-show: remove multiple tags from single message" -test_emacs "(notmuch-show \"$os_x_darwin_thread\") - (execute-kbd-macro \"-tag1-from-show-view -tag2-from-show-view\")" -output=$(notmuch search $os_x_darwin_thread | notmuch_search_sanitize) -test_expect_equal "$output" "thread:XXX 2009-11-18 [4/4] Jjgod Jiang, Alexander Botero-Lowry; [notmuch] Mac OS X/Darwin compatibility issues (inbox unread)" - -test_begin_subtest "notmuch-show: before-tag-hook is run, variables are defined" -output=$(test_emacs '(let ((notmuch-test-tag-hook-output nil) - (notmuch-before-tag-hook (function notmuch-test-tag-hook))) - (notmuch-show "id:ddd65cda0911171950o4eea4389v86de9525e46052d3@mail.gmail.com") - (execute-kbd-macro "+activate-hook\n") - (execute-kbd-macro "-activate-hook\n") - notmuch-test-tag-hook-output)') -test_expect_equal "$output" \ -'(("id:ddd65cda0911171950o4eea4389v86de9525e46052d3@mail.gmail.com" "-activate-hook") - ("id:ddd65cda0911171950o4eea4389v86de9525e46052d3@mail.gmail.com" "+activate-hook"))' - -test_begin_subtest "notmuch-show: after-tag-hook is run, variables are defined" -output=$(test_emacs '(let ((notmuch-test-tag-hook-output nil) - (notmuch-after-tag-hook (function notmuch-test-tag-hook))) - (notmuch-show "id:ddd65cda0911171950o4eea4389v86de9525e46052d3@mail.gmail.com") - (execute-kbd-macro "+activate-hook\n") - (execute-kbd-macro "-activate-hook\n") - notmuch-test-tag-hook-output)') -test_expect_equal "$output" \ -'(("id:ddd65cda0911171950o4eea4389v86de9525e46052d3@mail.gmail.com" "-activate-hook") - ("id:ddd65cda0911171950o4eea4389v86de9525e46052d3@mail.gmail.com" "+activate-hook"))' - test_begin_subtest "Message with .. in Message-Id:" add_message [id]=123..456@example '[subject]="Message with .. in Message-Id"' test_emacs '(notmuch-search "id:\"123..456@example\"") @@ -1133,30 +1064,6 @@ This is a warning (see *Notmuch errors* for more details) This is a warning This is another warning" -test_begin_subtest "Search thread tag operations are race-free" -add_message '[subject]="Search race test"' -gen_msg_id_1=$gen_msg_id -generate_message '[in-reply-to]="<'$gen_msg_id_1'>"' \ - '[references]="<'$gen_msg_id_1'>"' \ - '[subject]="Search race test two"' -test_emacs '(notmuch-search "subject:\"search race test\"") - (notmuch-test-wait) - (notmuch-poll) - (execute-kbd-macro "+search-thread-race-tag")' -output=$(notmuch search --output=messages 'tag:search-thread-race-tag') -test_expect_equal "$output" "id:$gen_msg_id_1" - -test_begin_subtest "Search global tag operations are race-free" -generate_message '[in-reply-to]="<'$gen_msg_id_1'>"' \ - '[references]="<'$gen_msg_id_1'>"' \ - '[subject]="Re: Search race test"' -test_emacs '(notmuch-search "subject:\"search race test\" -subject:two") - (notmuch-test-wait) - (notmuch-poll) - (execute-kbd-macro "*+search-global-race-tag")' -output=$(notmuch search --output=messages 'tag:search-global-race-tag') -test_expect_equal "$output" "id:$gen_msg_id_1" - test_begin_subtest "Term escaping" output=$(test_emacs "(mapcar 'notmuch-escape-boolean-term (list \"\" diff --git a/test/T315-emacs-tagging.sh b/test/T315-emacs-tagging.sh new file mode 100755 index 00000000..c26413ce --- /dev/null +++ b/test/T315-emacs-tagging.sh @@ -0,0 +1,166 @@ +#!/usr/bin/env bash + +test_description="emacs interface" +. $(dirname "$0")/test-lib.sh || exit 1 +. $NOTMUCH_SRCDIR/test/test-lib-emacs.sh || exit 1 + +EXPECTED=$NOTMUCH_SRCDIR/test/emacs.expected-output + +test_require_emacs +add_email_corpus + +test_begin_subtest "Add tag from search view" +os_x_darwin_thread=$(notmuch search --output=threads id:ddd65cda0911171950o4eea4389v86de9525e46052d3@mail.gmail.com) +test_emacs "(notmuch-search \"$os_x_darwin_thread\") + (notmuch-test-wait) + (execute-kbd-macro \"+tag-from-search-view\")" +output=$(notmuch search $os_x_darwin_thread | notmuch_search_sanitize) +test_expect_equal "$output" "thread:XXX 2009-11-18 [4/4] Jjgod Jiang, Alexander Botero-Lowry; [notmuch] Mac OS X/Darwin compatibility issues (inbox tag-from-search-view unread)" + +test_begin_subtest "Remove tag from search view" +test_emacs "(notmuch-search \"$os_x_darwin_thread\") + (notmuch-test-wait) + (execute-kbd-macro \"-tag-from-search-view\")" +output=$(notmuch search $os_x_darwin_thread | notmuch_search_sanitize) +test_expect_equal "$output" "thread:XXX 2009-11-18 [4/4] Jjgod Jiang, Alexander Botero-Lowry; [notmuch] Mac OS X/Darwin compatibility issues (inbox unread)" + +test_begin_subtest "Add tag (large query)" +# We use a long query to force us into batch mode and use a funny tag +# that requires escaping for batch tagging. +test_emacs "(notmuch-tag (concat \"$os_x_darwin_thread\" \" or \" (mapconcat #'identity (make-list notmuch-tag-argument-limit \"x\") \"-\")) (list \"+tag-from-%-large-query\"))" +output=$(notmuch search $os_x_darwin_thread | notmuch_search_sanitize) +test_expect_equal "$output" "thread:XXX 2009-11-18 [4/4] Jjgod Jiang, Alexander Botero-Lowry; [notmuch] Mac OS X/Darwin compatibility issues (inbox tag-from-%-large-query unread)" +notmuch tag -tag-from-%-large-query $os_x_darwin_thread + +test_begin_subtest "notmuch-show: add single tag to single message" +test_emacs "(notmuch-show \"$os_x_darwin_thread\") + (execute-kbd-macro \"+tag-from-show-view\")" +output=$(notmuch search $os_x_darwin_thread | notmuch_search_sanitize) +test_expect_equal "$output" "thread:XXX 2009-11-18 [4/4] Jjgod Jiang, Alexander Botero-Lowry; [notmuch] Mac OS X/Darwin compatibility issues (inbox tag-from-show-view unread)" + +test_begin_subtest "notmuch-show: remove single tag from single message" +test_emacs "(notmuch-show \"$os_x_darwin_thread\") + (execute-kbd-macro \"-tag-from-show-view\")" +output=$(notmuch search $os_x_darwin_thread | notmuch_search_sanitize) +test_expect_equal "$output" "thread:XXX 2009-11-18 [4/4] Jjgod Jiang, Alexander Botero-Lowry; [notmuch] Mac OS X/Darwin compatibility issues (inbox unread)" + +test_begin_subtest "notmuch-show: add multiple tags to single message" +test_emacs "(notmuch-show \"$os_x_darwin_thread\") + (execute-kbd-macro \"+tag1-from-show-view +tag2-from-show-view\")" +output=$(notmuch search $os_x_darwin_thread | notmuch_search_sanitize) +test_expect_equal "$output" "thread:XXX 2009-11-18 [4/4] Jjgod Jiang, Alexander Botero-Lowry; [notmuch] Mac OS X/Darwin compatibility issues (inbox tag1-from-show-view tag2-from-show-view unread)" + +test_begin_subtest "notmuch-show: remove multiple tags from single message" +test_emacs "(notmuch-show \"$os_x_darwin_thread\") + (execute-kbd-macro \"-tag1-from-show-view -tag2-from-show-view\")" +output=$(notmuch search $os_x_darwin_thread | notmuch_search_sanitize) +test_expect_equal "$output" "thread:XXX 2009-11-18 [4/4] Jjgod Jiang, Alexander Botero-Lowry; [notmuch] Mac OS X/Darwin compatibility issues (inbox unread)" + +test_begin_subtest "notmuch-show: before-tag-hook is run, variables are defined" +output=$(test_emacs '(let ((notmuch-test-tag-hook-output nil) + (notmuch-before-tag-hook (function notmuch-test-tag-hook))) + (notmuch-show "id:ddd65cda0911171950o4eea4389v86de9525e46052d3@mail.gmail.com") + (execute-kbd-macro "+activate-hook\n") + (execute-kbd-macro "-activate-hook\n") + notmuch-test-tag-hook-output)') +test_expect_equal "$output" \ +'(("id:ddd65cda0911171950o4eea4389v86de9525e46052d3@mail.gmail.com" "-activate-hook") + ("id:ddd65cda0911171950o4eea4389v86de9525e46052d3@mail.gmail.com" "+activate-hook"))' + +test_begin_subtest "notmuch-show: after-tag-hook is run, variables are defined" +output=$(test_emacs '(let ((notmuch-test-tag-hook-output nil) + (notmuch-after-tag-hook (function notmuch-test-tag-hook))) + (notmuch-show "id:ddd65cda0911171950o4eea4389v86de9525e46052d3@mail.gmail.com") + (execute-kbd-macro "+activate-hook\n") + (execute-kbd-macro "-activate-hook\n") + notmuch-test-tag-hook-output)') +test_expect_equal "$output" \ +'(("id:ddd65cda0911171950o4eea4389v86de9525e46052d3@mail.gmail.com" "-activate-hook") + ("id:ddd65cda0911171950o4eea4389v86de9525e46052d3@mail.gmail.com" "+activate-hook"))' + + +test_begin_subtest "Search thread tag operations are race-free" +add_message '[subject]="Search race test"' +gen_msg_id_1=$gen_msg_id +generate_message '[in-reply-to]="<'$gen_msg_id_1'>"' \ + '[references]="<'$gen_msg_id_1'>"' \ + '[subject]="Search race test two"' +test_emacs '(notmuch-search "subject:\"search race test\"") + (notmuch-test-wait) + (notmuch-poll) + (execute-kbd-macro "+search-thread-race-tag")' +output=$(notmuch search --output=messages 'tag:search-thread-race-tag') +test_expect_equal "$output" "id:$gen_msg_id_1" + +test_begin_subtest "Search global tag operations are race-free" +generate_message '[in-reply-to]="<'$gen_msg_id_1'>"' \ + '[references]="<'$gen_msg_id_1'>"' \ + '[subject]="Re: Search race test"' +test_emacs '(notmuch-search "subject:\"search race test\" -subject:two") + (notmuch-test-wait) + (notmuch-poll) + (execute-kbd-macro "*+search-global-race-tag")' +output=$(notmuch search --output=messages 'tag:search-global-race-tag') +test_expect_equal "$output" "id:$gen_msg_id_1" + +test_begin_subtest "undo with empty history is an error" +test_emacs "(let ((notmuch-tag-history nil)) + (test-log-error + (notmuch-tag-undo))) + " +cat < EXPECTED +(error no further notmuch undo information) +EOF +test_expect_equal_file EXPECTED MESSAGES + +for mode in search show tree unthreaded; do + test_begin_subtest "undo tagging in $mode mode" + test_emacs "(let ((notmuch-tag-history nil)) + (notmuch-$mode \"$os_x_darwin_thread\") + (notmuch-test-wait) + (execute-kbd-macro \"+tag-to-be-undone-$mode\") + (notmuch-tag-undo) + (notmuch-test-wait))" + count=$(notmuch count "tag:tag-to-be-undone-$mode") + test_expect_equal "$count" "0" + + test_begin_subtest "undo tagging in $mode mode (multiple operations)" + test_emacs "(let ((notmuch-tag-history nil)) + (notmuch-$mode \"$os_x_darwin_thread\") + (notmuch-test-wait) + (execute-kbd-macro \"+one-$mode\") + (execute-kbd-macro \"+two-$mode\") + (notmuch-tag-undo) + (notmuch-test-wait) + (execute-kbd-macro \"+three-$mode\"))" + output=$(notmuch search $os_x_darwin_thread | notmuch_search_sanitize) + notmuch tag "-one-$mode" "-three-$mode" $os_x_darwin_thread + test_expect_equal "$output" "thread:XXX 2009-11-18 [4/4] Jjgod Jiang, Alexander Botero-Lowry; [notmuch] Mac OS X/Darwin compatibility issues (inbox one-$mode three-$mode unread)" + + test_begin_subtest "undo tagging in $mode mode (multiple undo)" + test_emacs "(let ((notmuch-tag-history nil)) + (notmuch-$mode \"$os_x_darwin_thread\") + (notmuch-test-wait) + (execute-kbd-macro \"+one-$mode\") + (execute-kbd-macro \"+two-$mode\") + (notmuch-tag-undo) + (notmuch-test-wait) + (notmuch-tag-undo) + (notmuch-test-wait) + (execute-kbd-macro \"+three-$mode\"))" + output=$(notmuch search $os_x_darwin_thread | notmuch_search_sanitize) + notmuch tag "-one-$mode" "-three-$mode" $os_x_darwin_thread + test_expect_equal "$output" "thread:XXX 2009-11-18 [4/4] Jjgod Jiang, Alexander Botero-Lowry; [notmuch] Mac OS X/Darwin compatibility issues (inbox three-$mode unread)" + + test_begin_subtest "undo tagging in $mode mode (via binding)" + test_emacs "(let ((notmuch-tag-history nil)) + (notmuch-$mode \"$os_x_darwin_thread\") + (notmuch-test-wait) + (execute-kbd-macro \"+tag-to-be-undone-$mode\") + (execute-kbd-macro (kbd \"C-x u\")) + (notmuch-test-wait))" + count=$(notmuch count "tag:tag-to-be-undone-$mode") + test_expect_equal "$count" "0" +done + +test_done diff --git a/test/T355-smime.sh b/test/T355-smime.sh index 31fa4b4e..809274ce 100755 --- a/test/T355-smime.sh +++ b/test/T355-smime.sh @@ -35,6 +35,11 @@ EOF test_expect_equal_file EXPECTED OUTPUT test_begin_subtest "signature verification (notmuch CLI)" +if [ $NOTMUCH_GMIME_EMITS_ANGLE_BRACKETS == 1 ]; then + EXPECTED_EMAIL_ADDR='' +else + EXPECTED_EMAIL_ADDR='test_suite@notmuchmail.org' +fi output=$(notmuch show --format=json --verify subject:"test signed message 001" \ | notmuch_json_show_sanitize \ | sed -e 's|"created": [-1234567890]*|"created": 946728000|g' \ @@ -46,7 +51,7 @@ expected='[[[{"id": "XXXXX", "timestamp": 946728000, "date_relative": "2000-01-01", "tags": ["inbox","signed"], - "crypto": {"signed": {"status": [{"fingerprint": "'$FINGERPRINT'", "status": "good","userid": "CN=Notmuch Test Suite", "email": "", "expires": 424242424, "created": 946728000}]}}, + "crypto": {"signed": {"status": [{"fingerprint": "'$FINGERPRINT'", "status": "good","userid": "CN=Notmuch Test Suite", "email": "'$EXPECTED_EMAIL_ADDR'", "expires": 424242424, "created": 946728000}]}}, "headers": {"Subject": "test signed message 001", "From": "Notmuch Test Suite ", "To": "test_suite@notmuchmail.org", @@ -55,7 +60,7 @@ expected='[[[{"id": "XXXXX", "sigstatus": [{"fingerprint": "'$FINGERPRINT'", "status": "good", "userid": "CN=Notmuch Test Suite", - "email": "", + "email": "'$EXPECTED_EMAIL_ADDR'", "expires": 424242424, "created": 946728000}], "content-type": "multipart/signed", diff --git a/test/T380-atomicity.sh b/test/T380-atomicity.sh index afe49d93..0f9e6d2e 100755 --- a/test/T380-atomicity.sh +++ b/test/T380-atomicity.sh @@ -71,8 +71,8 @@ if test_require_external_prereq gdb; then # Check output against golden output outcount=$(cat outcount) - echo -n > searchall - echo -n > expectall + : > searchall + : > expectall for ((i = 0; i < $outcount; i++)); do if ! cmp -s search.$i expected; then # Find the range of interruptions that match this output diff --git a/test/T650-regexp-query.sh b/test/T650-regexp-query.sh index 55dc6c88..a9844501 100755 --- a/test/T650-regexp-query.sh +++ b/test/T650-regexp-query.sh @@ -65,6 +65,31 @@ thread:XXX 2001-01-05 [1/1] Notmuch Test Suite; - (inbox unread) EOF test_expect_equal_file EXPECTED OUTPUT +test_begin_subtest "bracketed subject search (with dquotes)" +notmuch search subject:notmuch and subject:show > EXPECTED +notmuch search 'subject:"(show notmuch)"' > OUTPUT +test_expect_equal_file_nonempty EXPECTED OUTPUT + +test_begin_subtest "bracketed subject search (with dquotes and operator 'or')" +notmuch search subject:notmuch or subject:show > EXPECTED +notmuch search 'subject:"(notmuch or show)"' > OUTPUT +test_expect_equal_file_nonempty EXPECTED OUTPUT + +test_begin_subtest "bracketed subject search (with dquotes and operator 'and')" +notmuch search subject:notmuch and subject:show > EXPECTED +notmuch search 'subject:"(notmuch and show)"' > OUTPUT +test_expect_equal_file_nonempty EXPECTED OUTPUT + +test_begin_subtest "bracketed subject search (with phrase, operator 'or')" +notmuch search 'subject:"mailing list"' or subject:FreeBSD > EXPECTED +notmuch search 'subject:"(""mailing list"" or FreeBSD)"' > OUTPUT +test_expect_equal_file_nonempty EXPECTED OUTPUT + +test_begin_subtest "bracketed subject search (with phrase, operator 'and')" +notmuch search search 'subject:"notmuch show"' and subject:commands > EXPECTED +notmuch search 'subject:"(""notmuch show"" and commands)"' > OUTPUT +test_expect_equal_file_nonempty EXPECTED OUTPUT + test_begin_subtest "xapian wildcard search for from:" notmuch search --output=messages 'from:cwo*' > OUTPUT test_expect_equal_file cworth.msg-ids OUTPUT diff --git a/test/corpora/indexing/mbox-attachment.eml b/test/corpora/indexing/mbox-attachment.eml new file mode 100644 index 00000000..98a8fc91 --- /dev/null +++ b/test/corpora/indexing/mbox-attachment.eml @@ -0,0 +1,83 @@ +From david@tethera.net Sat Feb 5 09:19:10 2022 +From: David Bremner +To: David Bremner +Subject: Re: [RFC PATCH v2 12/12] emacs: whitespace cleanup for keybindings +Date: Sat, 05 Feb 2022 10:19:09 -0400 +Message-ID: <87k0e9o0pu.fsf@tethera.net> +MIME-Version: 1.0 +Content-Type: multipart/mixed; boundary="=-=-=" + +--=-=-= +Content-Type: text/plain +Content-Disposition: inline + + +I figured out the race condition in the tests. The previous test was +still running when the failing test started, the joys of using a shared +emacs for running all of the tests in one file. + +The attached diff is split into the the commits that introduce the tests +in question in my working series, but you should be able to just apply +it on top of the posted series if you want. + + +--=-=-= +Content-Type: text/x-diff +Content-Disposition: inline; filename=0001-test-fixups.patch + +From fc88cba7f1f37b9cf3b296eace2422dd0e173502 Mon Sep 17 00:00:00 2001 +From: David Bremner +Date: Thu, 3 Feb 2022 21:05:05 -0400 +Subject: [PATCH] test fixups + +--- + test/T315-emacs-tagging.sh | 9 ++++----- + 1 file changed, 4 insertions(+), 5 deletions(-) + +diff --git a/test/T315-emacs-tagging.sh b/test/T315-emacs-tagging.sh +index c9e3e53a..c26413ce 100755 +--- a/test/T315-emacs-tagging.sh ++++ b/test/T315-emacs-tagging.sh +@@ -119,7 +119,8 @@ for mode in search show tree unthreaded; do + (notmuch-$mode \"$os_x_darwin_thread\") + (notmuch-test-wait) + (execute-kbd-macro \"+tag-to-be-undone-$mode\") +- (notmuch-tag-undo))" ++ (notmuch-tag-undo) ++ (notmuch-test-wait))" + count=$(notmuch count "tag:tag-to-be-undone-$mode") + test_expect_equal "$count" "0" + +@@ -128,9 +129,7 @@ for mode in search show tree unthreaded; do + (notmuch-$mode \"$os_x_darwin_thread\") + (notmuch-test-wait) + (execute-kbd-macro \"+one-$mode\") +- (notmuch-test-wait) + (execute-kbd-macro \"+two-$mode\") +- (notmuch-test-wait) + (notmuch-tag-undo) + (notmuch-test-wait) + (execute-kbd-macro \"+three-$mode\"))" +@@ -143,7 +142,6 @@ for mode in search show tree unthreaded; do + (notmuch-$mode \"$os_x_darwin_thread\") + (notmuch-test-wait) + (execute-kbd-macro \"+one-$mode\") +- (notmuch-test-wait) + (execute-kbd-macro \"+two-$mode\") + (notmuch-tag-undo) + (notmuch-test-wait) +@@ -159,7 +157,8 @@ for mode in search show tree unthreaded; do + (notmuch-$mode \"$os_x_darwin_thread\") + (notmuch-test-wait) + (execute-kbd-macro \"+tag-to-be-undone-$mode\") +- (execute-kbd-macro (kbd \"C-x u\")))" ++ (execute-kbd-macro (kbd \"C-x u\")) ++ (notmuch-test-wait))" + count=$(notmuch count "tag:tag-to-be-undone-$mode") + test_expect_equal "$count" "0" + done +-- +2.30.2 + + +--=-=-=-- diff --git a/test/test-lib-common.sh b/test/test-lib-common.sh index ebbf4cdf..18fa29c0 100644 --- a/test/test-lib-common.sh +++ b/test/test-lib-common.sh @@ -29,6 +29,20 @@ if [[ -z "$NOTMUCH_SRCDIR" ]] || [[ -z "$NOTMUCH_BUILDDIR" ]]; then exit 1 fi +# Explicitly require external prerequisite. Useful when binary is +# called indirectly (e.g. from emacs). +# Returns success if dependency is available, failure otherwise. +test_require_external_prereq () { + local binary + binary="$1" + if [[ ${test_missing_external_prereq_["${binary}"]} == t ]]; then + # dependency is missing, call the replacement function to note it + eval "$binary" + else + true + fi +} + backup_database () { test_name=$(basename $0 .sh) rm -rf $TMP_DIRECTORY/notmuch-dir-backup."$test_name" diff --git a/test/test-lib-emacs.sh b/test/test-lib-emacs.sh index a298526d..ad4c4aeb 100644 --- a/test/test-lib-emacs.sh +++ b/test/test-lib-emacs.sh @@ -207,4 +207,12 @@ test_emacs () { ${TEST_EMACSCLIENT} --socket-name="$EMACS_SERVER" --eval "(notmuch-test-progn $*)" } +time_emacs () { + rm -f MESSAGES + printf "%s" "$1" + shift + test_emacs "(test-time $*)" > emacs.out + tail -n 1 MESSAGES +} + emacs_generate_script diff --git a/test/test-lib.el b/test/test-lib.el index 6831b46f..79a9d4d6 100644 --- a/test/test-lib.el +++ b/test/test-lib.el @@ -186,6 +186,11 @@ running, quit if it terminated." (t (message "%s" err))) (with-current-buffer "*Messages*" (test-output "MESSAGES")))) +(defmacro test-time (&rest body) + `(let ((results (mapcar (lambda (x) (/ x 5.0)) (benchmark-run 5 ,@body)))) + (message "\t\t%0.2f\t%0.2f\t%0.2f" (nth 0 results) (nth 1 results) (nth 2 results)) + (with-current-buffer "*Messages*" (test-output "MESSAGES")))) + ;; For historical reasons, we hide deleted tags by default in the test ;; suite (setq notmuch-tag-deleted-formats diff --git a/test/test-lib.sh b/test/test-lib.sh index 833bf5fe..59b6079d 100644 --- a/test/test-lib.sh +++ b/test/test-lib.sh @@ -64,55 +64,7 @@ exec 6>&1 7>&2 BASH_XTRACEFD=7 export PS4='+(${BASH_SOURCE}:${LINENO}): ${FUNCNAME[0]:+${FUNCNAME[0]}(): }' -# Keep the original TERM for say_color and test_emacs -ORIGINAL_TERM=$TERM - -# Set SMART_TERM to vt100 for known dumb/unknown terminal. -# Otherwise use whatever TERM is currently used so that -# users' actual TERM environments are being used in tests. -case ${TERM-} in - '' | dumb | unknown ) - SMART_TERM=vt100 ;; - *) - SMART_TERM=$TERM ;; -esac - -# For repeatability, reset the environment to known value. -LANG=C -LC_ALL=C -PAGER=cat -TZ=UTC -TERM=dumb -export LANG LC_ALL PAGER TERM TZ -GIT_TEST_CMP=${GIT_TEST_CMP:-diff -u} -if [[ ( -n "$TEST_EMACS" && -z "$TEST_EMACSCLIENT" ) || \ - ( -z "$TEST_EMACS" && -n "$TEST_EMACSCLIENT" ) ]]; then - echo "error: must specify both or neither of TEST_EMACS and TEST_EMACSCLIENT" >&2 - exit 1 -fi -TEST_EMACS=${TEST_EMACS:-${EMACS:-emacs}} -TEST_EMACSCLIENT=${TEST_EMACSCLIENT:-emacsclient} -TEST_GDB=${TEST_GDB:-gdb} -TEST_CC=${TEST_CC:-cc} -TEST_CFLAGS=${TEST_CFLAGS:-"-g -O0"} -TEST_SHIM_CFLAGS=${TEST_SHIM_CFLAGS:-"-fpic -shared"} -TEST_SHIM_LDFLAGS=${TEST_SHIM_LDFLAGS:-"-ldl"} - -# Protect ourselves from common misconfiguration to export -# CDPATH into the environment -unset CDPATH - -unset GREP_OPTIONS - -# For lib/open.cc:_load_key_file -unset XDG_CONFIG_HOME - -# For emacsclient -unset ALTERNATE_EDITOR - -# for reproducibility -unset EMAIL -unset NAME +. "$NOTMUCH_SRCDIR/test/test-vars.sh" || exit 1 add_gnupg_home () { [ -e "${GNUPGHOME}/gpg.conf" ] && return @@ -330,11 +282,6 @@ die () { exit 1 } -GIT_EXIT_OK= -# Note: TEST_TMPDIR *NOT* exported! -TEST_TMPDIR=$(mktemp -d "${TMPDIR:-/tmp}/notmuch-test-$$.XXXXXX") -# Put GNUPGHOME in TMPDIR to avoid problems with long paths. -export GNUPGHOME="${TEST_TMPDIR}/gnupg" trap 'trap_exit' EXIT trap 'trap_signal' HUP INT TERM @@ -656,20 +603,6 @@ $binary () { fi } -# Explicitly require external prerequisite. Useful when binary is -# called indirectly (e.g. from emacs). -# Returns success if dependency is available, failure otherwise. -test_require_external_prereq () { - local binary - binary="$1" - if [[ ${test_missing_external_prereq_["${binary}"]} == t ]]; then - # dependency is missing, call the replacement function to note it - eval "$binary" - else - true - fi -} - # You are not expected to call test_ok_ and test_failure_ directly, use # the text_expect_* functions instead. diff --git a/test/test-vars.sh b/test/test-vars.sh new file mode 100644 index 00000000..02d60f89 --- /dev/null +++ b/test/test-vars.sh @@ -0,0 +1,62 @@ +# Common variable settings for (correctness) tests and performance +# tests. + +# Keep the original TERM for say_color and test_emacs +ORIGINAL_TERM=$TERM + +# Set SMART_TERM to vt100 for known dumb/unknown terminal. +# Otherwise use whatever TERM is currently used so that +# users' actual TERM environments are being used in tests. +case ${TERM-} in + '' | dumb | unknown ) + SMART_TERM=vt100 ;; + *) + SMART_TERM=$TERM ;; +esac + +# For repeatability, reset the environment to known value. +LANG=C +LC_ALL=C +PAGER=cat +TZ=UTC +TERM=dumb +export LANG LC_ALL PAGER TERM TZ +GIT_TEST_CMP=${GIT_TEST_CMP:-diff -u} +if [[ ( -n "$TEST_EMACS" && -z "$TEST_EMACSCLIENT" ) || \ + ( -z "$TEST_EMACS" && -n "$TEST_EMACSCLIENT" ) ]]; then + echo "error: must specify both or neither of TEST_EMACS and TEST_EMACSCLIENT" >&2 + exit 1 +fi +TEST_EMACS=${TEST_EMACS:-${EMACS:-emacs}} +TEST_EMACSCLIENT=${TEST_EMACSCLIENT:-emacsclient} +TEST_GDB=${TEST_GDB:-gdb} +TEST_CC=${TEST_CC:-cc} +TEST_CFLAGS=${TEST_CFLAGS:-"-g -O0"} +TEST_SHIM_CFLAGS=${TEST_SHIM_CFLAGS:-"-fpic -shared"} +TEST_SHIM_LDFLAGS=${TEST_SHIM_LDFLAGS:-"-ldl"} + +# Protect ourselves from common misconfiguration to export +# CDPATH into the environment +unset CDPATH + +unset GREP_OPTIONS + +# For lib/open.cc:_load_key_file +unset XDG_CONFIG_HOME + +# for lib/open.cc:_choose_database_path +unset XDG_DATA_HOME +unset MAILDIR + +# For emacsclient +unset ALTERNATE_EDITOR + +# for reproducibility +unset EMAIL +unset NAME + +GIT_EXIT_OK= +# Note: TEST_TMPDIR *NOT* exported! +TEST_TMPDIR=$(mktemp -d "${TMPDIR:-/tmp}/notmuch-test-$$.XXXXXX") +# Put GNUPGHOME in TMPDIR to avoid problems with long paths. +export GNUPGHOME="${TEST_TMPDIR}/gnupg" diff --git a/version.txt b/version.txt index c74e8a04..2037dfa6 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -0.35 +0.36