1 /* database.cc - The database interfaces of the notmuch mail library
3 * Copyright © 2009 Carl Worth
5 * This program is free software: you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation, either version 3 of the License, or
8 * (at your option) any later version.
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
15 * You should have received a copy of the GNU General Public License
16 * along with this program. If not, see https://www.gnu.org/licenses/ .
18 * Author: Carl Worth <cworth@cworth.org>
21 #include "database-private.h"
22 #include "parse-time-vrp.h"
24 #include "thread-fp.h"
25 #include "regexp-fields.h"
26 #include "string-util.h"
35 #include <glib.h> /* g_free, GPtrArray, GHashTable */
36 #include <glib-object.h> /* g_type_init */
38 #include <gmime/gmime.h> /* g_mime_init */
42 #define ARRAY_SIZE(arr) (sizeof (arr) / sizeof (arr[0]))
47 notmuch_field_flag_t flags;
50 #define NOTMUCH_DATABASE_VERSION 3
52 #define STRINGIFY(s) _SUB_STRINGIFY(s)
53 #define _SUB_STRINGIFY(s) #s
55 #if HAVE_XAPIAN_DB_RETRY_LOCK
56 #define DB_ACTION (Xapian::DB_CREATE_OR_OPEN | Xapian::DB_RETRY_LOCK)
58 #define DB_ACTION Xapian::DB_CREATE_OR_OPEN
61 /* Here's the current schema for our database (for NOTMUCH_DATABASE_VERSION):
63 * We currently have three different types of documents (mail, ghost,
64 * and directory) and also some metadata.
66 * There are two kinds of prefixes used in notmuch. There are the
67 * human friendly 'prefix names' like "thread:", which are also used
68 * in the query parser, and the actual prefix terms in the database
69 * (e.g. "G"). The correspondence is maintained in the file scope data
70 * structure 'prefix_table'.
74 * A mail document is associated with a particular email message. It
75 * is stored in one or more files on disk and is uniquely identified
76 * by its "id" field (which is generally the message ID). It is
77 * indexed with the following prefixed terms which the database uses
78 * to construct threads, etc.:
80 * Single terms of given prefix:
84 * id: Unique ID of mail. This is from the Message-ID header
85 * if present and not too long (see NOTMUCH_MESSAGE_ID_MAX).
86 * If it's present and too long, then we use
87 * "notmuch-sha1-<sha1_sum_of_message_id>".
88 * If this header is not present, we use
89 * "notmuch-sha1-<sha1_sum_of_entire_file>".
91 * thread: The ID of the thread to which the mail belongs
93 * replyto: The ID from the In-Reply-To header of the mail (if any).
95 * Multiple terms of given prefix:
97 * reference: All message IDs from In-Reply-To and References
98 * headers in the message.
100 * tag: Any tags associated with this message by the user.
102 * file-direntry: A colon-separated pair of values
103 * (INTEGER:STRING), where INTEGER is the
104 * document ID of a directory document, and
105 * STRING is the name of a file within that
106 * directory for this mail message.
108 * property: Has a property with key=value
109 * FIXME: if no = is present, should match on any value
111 * A mail document also has four values:
113 * TIMESTAMP: The time_t value corresponding to the message's
116 * MESSAGE_ID: The unique ID of the mail mess (see "id" above)
118 * FROM: The value of the "From" header
120 * SUBJECT: The value of the "Subject" header
122 * LAST_MOD: The revision number as of the last tag or
125 * The prefixed terms described above are also searchable without an
126 * explicit field name, but as of notmuch 0.29 this is due to
127 * query-parser setup, not extra terms in the database. In addition,
128 * terms from the content of the message are added without a prefix
129 * for use by the user in searching. Note that the prefix name "body"
130 * is used to refer to the empty prefix string in the database.
132 * The path of the containing folder is added with the "folder" prefix
133 * (see _notmuch_message_add_folder_terms). Sub-paths of the the path
134 * of the mail message are added with the "path" prefix.
136 * The data portion of a mail document is empty.
138 * Ghost mail document [if NOTMUCH_FEATURE_GHOSTS]
139 * -----------------------------------------------
140 * A ghost mail document is like a mail document, but where we don't
141 * have the message content. These are used to track thread reference
142 * information for messages we haven't received.
144 * A ghost mail document has type: ghost; id and thread fields that
145 * are identical to the mail document fields; and a MESSAGE_ID value.
149 * A directory document is used by a client of the notmuch library to
150 * maintain data necessary to allow for efficient polling of mail
153 * All directory documents contain one term:
155 * directory: The directory path (relative to the database path)
156 * Or the SHA1 sum of the directory path (if the
157 * path itself is too long to fit in a Xapian
160 * And all directory documents for directories other than top-level
161 * directories also contain the following term:
163 * directory-direntry: A colon-separated pair of values
164 * (INTEGER:STRING), where INTEGER is the
165 * document ID of the parent directory
166 * document, and STRING is the name of this
167 * directory within that parent.
169 * All directory documents have a single value:
171 * TIMESTAMP: The mtime of the directory (at last scan)
173 * The data portion of a directory document contains the path of the
174 * directory (relative to the database path).
178 * Xapian allows us to store arbitrary name-value pairs as
179 * "metadata". We currently use the following metadata names with the
182 * version The database schema version, (which is distinct
183 * from both the notmuch package version (see
184 * notmuch --version) and the libnotmuch library
185 * version. The version is stored as an base-10
186 * ASCII integer. The initial database version
187 * was 1, (though a schema existed before that
188 * were no "version" database value existed at
189 * all). Successive versions are allocated as
190 * changes are made to the database (such as by
191 * indexing new fields).
193 * features The set of features supported by this
194 * database. This consists of a set of
195 * '\n'-separated lines, where each is a feature
196 * name, a '\t', and compatibility flags. If the
197 * compatibility flags contain 'w', then the
198 * opener must support this feature to safely
199 * write this database. If the compatibility
200 * flags contain 'r', then the opener must
201 * support this feature to read this database.
202 * Introduced in database version 3.
204 * last_thread_id The last thread ID generated. This is stored
205 * as a 16-byte hexadecimal ASCII representation
206 * of a 64-bit unsigned integer. The first ID
207 * generated is 1 and the value will be
208 * incremented for each thread ID.
210 * C* metadata keys starting with C indicate
211 * configuration data. It can be managed with the
212 * n_database_*config* API. There is a convention
213 * of hierarchical keys separated by '.' (e.g.
214 * query.notmuch stores the value for the named
215 * query 'notmuch'), but it is not enforced by the
221 * If ! NOTMUCH_FEATURE_GHOSTS, there are no ghost mail documents.
222 * Instead, the database has the following additional database
225 * thread_id_* A pre-allocated thread ID for a particular
226 * message. This is actually an arbitrarily large
227 * family of metadata name. Any particular name is
228 * formed by concatenating "thread_id_" with a message
229 * ID (or the SHA1 sum of a message ID if it is very
230 * long---see description of 'id' in the mail
231 * document). The value stored is a thread ID.
233 * These thread ID metadata values are stored
234 * whenever a message references a parent message
235 * that does not yet exist in the database. A
236 * thread ID will be allocated and stored, and if
237 * the message is later added, the stored thread
238 * ID will be used (and the metadata value will
241 * Even before a message is added, it's
242 * pre-allocated thread ID is useful so that all
243 * descendant messages that reference this common
244 * parent can be recognized as belonging to the
248 /* With these prefix values we follow the conventions published here:
250 * https://xapian.org/docs/omega/termprefixes.html
252 * as much as makes sense. Note that I took some liberty in matching
253 * the reserved prefix values to notmuch concepts, (for example, 'G'
254 * is documented as "newsGroup (or similar entity - e.g. a web forum
255 * name)", for which I think the thread is the closest analogue in
256 * notmuch. This in spite of the fact that we will eventually be
257 * storing mailing-list messages where 'G' for "mailing list name"
258 * might be even a closer analogue. I'm treating the single-character
259 * prefixes preferentially for core notmuch concepts (which will be
260 * nearly universal to all mail messages).
264 prefix_t prefix_table[] = {
265 /* name term prefix flags */
266 { "type", "T", NOTMUCH_FIELD_NO_FLAGS },
267 { "reference", "XREFERENCE", NOTMUCH_FIELD_NO_FLAGS },
268 { "replyto", "XREPLYTO", NOTMUCH_FIELD_NO_FLAGS },
269 { "directory", "XDIRECTORY", NOTMUCH_FIELD_NO_FLAGS },
270 { "file-direntry", "XFDIRENTRY", NOTMUCH_FIELD_NO_FLAGS },
271 { "directory-direntry", "XDDIRENTRY", NOTMUCH_FIELD_NO_FLAGS },
272 { "body", "", NOTMUCH_FIELD_EXTERNAL |
273 NOTMUCH_FIELD_PROBABILISTIC},
274 { "thread", "G", NOTMUCH_FIELD_EXTERNAL |
275 NOTMUCH_FIELD_PROCESSOR },
276 { "tag", "K", NOTMUCH_FIELD_EXTERNAL |
277 NOTMUCH_FIELD_PROCESSOR },
278 { "is", "K", NOTMUCH_FIELD_EXTERNAL |
279 NOTMUCH_FIELD_PROCESSOR },
280 { "id", "Q", NOTMUCH_FIELD_EXTERNAL },
281 { "mid", "Q", NOTMUCH_FIELD_EXTERNAL |
282 NOTMUCH_FIELD_PROCESSOR },
283 { "path", "P", NOTMUCH_FIELD_EXTERNAL|
284 NOTMUCH_FIELD_PROCESSOR },
285 { "property", "XPROPERTY", NOTMUCH_FIELD_EXTERNAL },
287 * Unconditionally add ':' to reduce potential ambiguity with
288 * overlapping prefixes and/or terms that start with capital
289 * letters. See Xapian document termprefixes.html for related
292 { "folder", "XFOLDER:", NOTMUCH_FIELD_EXTERNAL |
293 NOTMUCH_FIELD_PROCESSOR },
294 #if HAVE_XAPIAN_FIELD_PROCESSOR
295 { "date", NULL, NOTMUCH_FIELD_EXTERNAL |
296 NOTMUCH_FIELD_PROCESSOR },
297 { "query", NULL, NOTMUCH_FIELD_EXTERNAL |
298 NOTMUCH_FIELD_PROCESSOR },
300 { "from", "XFROM", NOTMUCH_FIELD_EXTERNAL |
301 NOTMUCH_FIELD_PROBABILISTIC |
302 NOTMUCH_FIELD_PROCESSOR },
303 { "to", "XTO", NOTMUCH_FIELD_EXTERNAL |
304 NOTMUCH_FIELD_PROBABILISTIC },
305 { "attachment", "XATTACHMENT", NOTMUCH_FIELD_EXTERNAL |
306 NOTMUCH_FIELD_PROBABILISTIC },
307 { "mimetype", "XMIMETYPE", NOTMUCH_FIELD_EXTERNAL |
308 NOTMUCH_FIELD_PROBABILISTIC },
309 { "subject", "XSUBJECT", NOTMUCH_FIELD_EXTERNAL |
310 NOTMUCH_FIELD_PROBABILISTIC |
311 NOTMUCH_FIELD_PROCESSOR},
315 _setup_query_field_default (const prefix_t *prefix, notmuch_database_t *notmuch)
318 notmuch->query_parser->add_prefix ("",prefix->prefix);
319 if (prefix->flags & NOTMUCH_FIELD_PROBABILISTIC)
320 notmuch->query_parser->add_prefix (prefix->name, prefix->prefix);
322 notmuch->query_parser->add_boolean_prefix (prefix->name, prefix->prefix);
325 #if HAVE_XAPIAN_FIELD_PROCESSOR
327 _setup_query_field (const prefix_t *prefix, notmuch_database_t *notmuch)
329 if (prefix->flags & NOTMUCH_FIELD_PROCESSOR) {
330 Xapian::FieldProcessor *fp;
332 if (STRNCMP_LITERAL (prefix->name, "date") == 0)
333 fp = (new DateFieldProcessor())->release ();
334 else if (STRNCMP_LITERAL(prefix->name, "query") == 0)
335 fp = (new QueryFieldProcessor (*notmuch->query_parser, notmuch))->release ();
336 else if (STRNCMP_LITERAL(prefix->name, "thread") == 0)
337 fp = (new ThreadFieldProcessor (*notmuch->query_parser, notmuch))->release ();
339 fp = (new RegexpFieldProcessor (prefix->name, prefix->flags,
340 *notmuch->query_parser, notmuch))->release ();
342 /* we treat all field-processor fields as boolean in order to get the raw input */
344 notmuch->query_parser->add_prefix ("",prefix->prefix);
345 notmuch->query_parser->add_boolean_prefix (prefix->name, fp);
347 _setup_query_field_default (prefix, notmuch);
352 _setup_query_field (const prefix_t *prefix, notmuch_database_t *notmuch)
354 _setup_query_field_default (prefix, notmuch);
359 _find_prefix (const char *name)
363 for (i = 0; i < ARRAY_SIZE (prefix_table); i++) {
364 if (strcmp (name, prefix_table[i].name) == 0)
365 return prefix_table[i].prefix;
368 INTERNAL_ERROR ("No prefix exists for '%s'\n", name);
373 static const struct {
374 /* NOTMUCH_FEATURE_* value. */
375 _notmuch_features value;
376 /* Feature name as it appears in the database. This name should
377 * be appropriate for displaying to the user if an older version
378 * of notmuch doesn't support this feature. */
380 /* Compatibility flags when this feature is declared. */
382 } feature_names[] = {
383 { NOTMUCH_FEATURE_FILE_TERMS,
384 "multiple paths per message", "rw" },
385 { NOTMUCH_FEATURE_DIRECTORY_DOCS,
386 "relative directory paths", "rw" },
387 /* Header values are not required for reading a database because a
388 * reader can just refer to the message file. */
389 { NOTMUCH_FEATURE_FROM_SUBJECT_ID_VALUES,
390 "from/subject/message-ID in database", "w" },
391 { NOTMUCH_FEATURE_BOOL_FOLDER,
392 "exact folder:/path: search", "rw" },
393 { NOTMUCH_FEATURE_GHOSTS,
394 "mail documents for missing messages", "w"},
395 /* Knowledge of the index mime-types are not required for reading
396 * a database because a reader will just be unable to query
398 { NOTMUCH_FEATURE_INDEXED_MIMETYPES,
399 "indexed MIME types", "w"},
400 { NOTMUCH_FEATURE_LAST_MOD,
401 "modification tracking", "w"},
402 /* Existing databases will work fine for all queries not involving
404 { NOTMUCH_FEATURE_UNPREFIX_BODY_ONLY,
405 "index body and headers separately", "w"},
409 notmuch_status_to_string (notmuch_status_t status)
412 case NOTMUCH_STATUS_SUCCESS:
413 return "No error occurred";
414 case NOTMUCH_STATUS_OUT_OF_MEMORY:
415 return "Out of memory";
416 case NOTMUCH_STATUS_READ_ONLY_DATABASE:
417 return "Attempt to write to a read-only database";
418 case NOTMUCH_STATUS_XAPIAN_EXCEPTION:
419 return "A Xapian exception occurred";
420 case NOTMUCH_STATUS_FILE_ERROR:
421 return "Something went wrong trying to read or write a file";
422 case NOTMUCH_STATUS_FILE_NOT_EMAIL:
423 return "File is not an email";
424 case NOTMUCH_STATUS_DUPLICATE_MESSAGE_ID:
425 return "Message ID is identical to a message in database";
426 case NOTMUCH_STATUS_NULL_POINTER:
427 return "Erroneous NULL pointer";
428 case NOTMUCH_STATUS_TAG_TOO_LONG:
429 return "Tag value is too long (exceeds NOTMUCH_TAG_MAX)";
430 case NOTMUCH_STATUS_UNBALANCED_FREEZE_THAW:
431 return "Unbalanced number of calls to notmuch_message_freeze/thaw";
432 case NOTMUCH_STATUS_UNBALANCED_ATOMIC:
433 return "Unbalanced number of calls to notmuch_database_begin_atomic/end_atomic";
434 case NOTMUCH_STATUS_UNSUPPORTED_OPERATION:
435 return "Unsupported operation";
436 case NOTMUCH_STATUS_UPGRADE_REQUIRED:
437 return "Operation requires a database upgrade";
438 case NOTMUCH_STATUS_PATH_ERROR:
439 return "Path supplied is illegal for this function";
440 case NOTMUCH_STATUS_MALFORMED_CRYPTO_PROTOCOL:
441 return "Crypto protocol missing, malformed, or unintelligible";
442 case NOTMUCH_STATUS_FAILED_CRYPTO_CONTEXT_CREATION:
443 return "Crypto engine initialization failure";
444 case NOTMUCH_STATUS_UNKNOWN_CRYPTO_PROTOCOL:
445 return "Unknown crypto protocol";
447 case NOTMUCH_STATUS_LAST_STATUS:
448 return "Unknown error status value";
453 _notmuch_database_log (notmuch_database_t *notmuch,
459 va_start (va_args, format);
461 if (notmuch->status_string)
462 talloc_free (notmuch->status_string);
464 notmuch->status_string = talloc_vasprintf (notmuch, format, va_args);
469 _notmuch_database_log_append (notmuch_database_t *notmuch,
475 va_start (va_args, format);
477 if (notmuch->status_string)
478 notmuch->status_string = talloc_vasprintf_append (notmuch->status_string, format, va_args);
480 notmuch->status_string = talloc_vasprintf (notmuch, format, va_args);
486 find_doc_ids_for_term (notmuch_database_t *notmuch,
488 Xapian::PostingIterator *begin,
489 Xapian::PostingIterator *end)
491 *begin = notmuch->xapian_db->postlist_begin (term);
493 *end = notmuch->xapian_db->postlist_end (term);
497 _notmuch_database_find_doc_ids (notmuch_database_t *notmuch,
498 const char *prefix_name,
500 Xapian::PostingIterator *begin,
501 Xapian::PostingIterator *end)
505 term = talloc_asprintf (notmuch, "%s%s",
506 _find_prefix (prefix_name), value);
508 find_doc_ids_for_term (notmuch, term, begin, end);
513 notmuch_private_status_t
514 _notmuch_database_find_unique_doc_id (notmuch_database_t *notmuch,
515 const char *prefix_name,
517 unsigned int *doc_id)
519 Xapian::PostingIterator i, end;
521 _notmuch_database_find_doc_ids (notmuch, prefix_name, value, &i, &end);
525 return NOTMUCH_PRIVATE_STATUS_NO_DOCUMENT_FOUND;
530 #if DEBUG_DATABASE_SANITY
534 INTERNAL_ERROR ("Term %s:%s is not unique as expected.\n",
538 return NOTMUCH_PRIVATE_STATUS_SUCCESS;
541 static Xapian::Document
542 find_document_for_doc_id (notmuch_database_t *notmuch, unsigned doc_id)
544 return notmuch->xapian_db->get_document (doc_id);
547 /* Generate a compressed version of 'message_id' of the form:
549 * notmuch-sha1-<sha1_sum_of_message_id>
552 _notmuch_message_id_compressed (void *ctx, const char *message_id)
554 char *sha1, *compressed;
556 sha1 = _notmuch_sha1_of_string (message_id);
558 compressed = talloc_asprintf (ctx, "notmuch-sha1-%s", sha1);
565 notmuch_database_find_message (notmuch_database_t *notmuch,
566 const char *message_id,
567 notmuch_message_t **message_ret)
569 notmuch_private_status_t status;
572 if (message_ret == NULL)
573 return NOTMUCH_STATUS_NULL_POINTER;
575 if (strlen (message_id) > NOTMUCH_MESSAGE_ID_MAX)
576 message_id = _notmuch_message_id_compressed (notmuch, message_id);
579 status = _notmuch_database_find_unique_doc_id (notmuch, "id",
580 message_id, &doc_id);
582 if (status == NOTMUCH_PRIVATE_STATUS_NO_DOCUMENT_FOUND)
585 *message_ret = _notmuch_message_create (notmuch, notmuch, doc_id,
587 if (*message_ret == NULL)
588 return NOTMUCH_STATUS_OUT_OF_MEMORY;
591 return NOTMUCH_STATUS_SUCCESS;
592 } catch (const Xapian::Error &error) {
593 _notmuch_database_log (notmuch, "A Xapian exception occurred finding message: %s.\n",
594 error.get_msg().c_str());
595 notmuch->exception_reported = true;
597 return NOTMUCH_STATUS_XAPIAN_EXCEPTION;
602 notmuch_database_create (const char *path, notmuch_database_t **database)
604 char *status_string = NULL;
605 notmuch_status_t status;
607 status = notmuch_database_create_verbose (path, database,
611 fputs (status_string, stderr);
612 free (status_string);
619 notmuch_database_create_verbose (const char *path,
620 notmuch_database_t **database,
621 char **status_string)
623 notmuch_status_t status = NOTMUCH_STATUS_SUCCESS;
624 notmuch_database_t *notmuch = NULL;
625 char *notmuch_path = NULL;
626 char *message = NULL;
631 message = strdup ("Error: Cannot create a database for a NULL path.\n");
632 status = NOTMUCH_STATUS_NULL_POINTER;
636 if (path[0] != '/') {
637 message = strdup ("Error: Database path must be absolute.\n");
638 status = NOTMUCH_STATUS_PATH_ERROR;
642 err = stat (path, &st);
644 IGNORE_RESULT (asprintf (&message, "Error: Cannot create database at %s: %s.\n",
645 path, strerror (errno)));
646 status = NOTMUCH_STATUS_FILE_ERROR;
650 if (! S_ISDIR (st.st_mode)) {
651 IGNORE_RESULT (asprintf (&message, "Error: Cannot create database at %s: "
652 "Not a directory.\n",
654 status = NOTMUCH_STATUS_FILE_ERROR;
658 notmuch_path = talloc_asprintf (NULL, "%s/%s", path, ".notmuch");
660 err = mkdir (notmuch_path, 0755);
663 IGNORE_RESULT (asprintf (&message, "Error: Cannot create directory %s: %s.\n",
664 notmuch_path, strerror (errno)));
665 status = NOTMUCH_STATUS_FILE_ERROR;
669 status = notmuch_database_open_verbose (path,
670 NOTMUCH_DATABASE_MODE_READ_WRITE,
675 /* Upgrade doesn't add these feature to existing databases, but
676 * new databases have them. */
677 notmuch->features |= NOTMUCH_FEATURE_FROM_SUBJECT_ID_VALUES;
678 notmuch->features |= NOTMUCH_FEATURE_INDEXED_MIMETYPES;
679 notmuch->features |= NOTMUCH_FEATURE_UNPREFIX_BODY_ONLY;
681 status = notmuch_database_upgrade (notmuch, NULL, NULL);
683 notmuch_database_close(notmuch);
689 talloc_free (notmuch_path);
693 *status_string = message;
700 talloc_free (notmuch);
705 _notmuch_database_ensure_writable (notmuch_database_t *notmuch)
707 if (notmuch->mode == NOTMUCH_DATABASE_MODE_READ_ONLY) {
708 _notmuch_database_log (notmuch, "Cannot write to a read-only database.\n");
709 return NOTMUCH_STATUS_READ_ONLY_DATABASE;
712 return NOTMUCH_STATUS_SUCCESS;
715 /* Allocate a revision number for the next change. */
717 _notmuch_database_new_revision (notmuch_database_t *notmuch)
719 unsigned long new_revision = notmuch->revision + 1;
721 /* If we're in an atomic section, hold off on updating the
722 * committed revision number until we commit the atomic section.
724 if (notmuch->atomic_nesting)
725 notmuch->atomic_dirty = true;
727 notmuch->revision = new_revision;
732 /* Parse a database features string from the given database version.
733 * Returns the feature bit set.
735 * For version < 3, this ignores the features string and returns a
736 * hard-coded set of features.
738 * If there are unrecognized features that are required to open the
739 * database in mode (which should be 'r' or 'w'), return a
740 * comma-separated list of unrecognized but required features in
741 * *incompat_out suitable for presenting to the user. *incompat_out
742 * will be allocated from ctx.
744 static _notmuch_features
745 _parse_features (const void *ctx, const char *features, unsigned int version,
746 char mode, char **incompat_out)
748 _notmuch_features res = static_cast<_notmuch_features>(0);
749 unsigned int namelen, i;
753 /* Prior to database version 3, features were implied by the
756 return NOTMUCH_FEATURES_V0;
757 else if (version == 1)
758 return NOTMUCH_FEATURES_V1;
759 else if (version == 2)
760 return NOTMUCH_FEATURES_V2;
762 /* Parse the features string */
763 while ((features = strtok_len_c (features + llen, "\n", &llen)) != NULL) {
764 flags = strchr (features, '\t');
765 if (! flags || flags > features + llen)
767 namelen = flags - features;
769 for (i = 0; i < ARRAY_SIZE (feature_names); ++i) {
770 if (strlen (feature_names[i].name) == namelen &&
771 strncmp (feature_names[i].name, features, namelen) == 0) {
772 res |= feature_names[i].value;
777 if (i == ARRAY_SIZE (feature_names) && incompat_out) {
778 /* Unrecognized feature */
779 const char *have = strchr (flags, mode);
780 if (have && have < features + llen) {
781 /* This feature is required to access this database in
782 * 'mode', but we don't understand it. */
784 *incompat_out = talloc_strdup (ctx, "");
785 *incompat_out = talloc_asprintf_append_buffer (
786 *incompat_out, "%s%.*s", **incompat_out ? ", " : "",
796 _print_features (const void *ctx, unsigned int features)
799 char *res = talloc_strdup (ctx, "");
801 for (i = 0; i < ARRAY_SIZE (feature_names); ++i)
802 if (features & feature_names[i].value)
803 res = talloc_asprintf_append_buffer (
804 res, "%s\t%s\n", feature_names[i].name, feature_names[i].flags);
810 notmuch_database_open (const char *path,
811 notmuch_database_mode_t mode,
812 notmuch_database_t **database)
814 char *status_string = NULL;
815 notmuch_status_t status;
817 status = notmuch_database_open_verbose (path, mode, database,
821 fputs (status_string, stderr);
822 free (status_string);
829 notmuch_database_open_verbose (const char *path,
830 notmuch_database_mode_t mode,
831 notmuch_database_t **database,
832 char **status_string)
834 notmuch_status_t status = NOTMUCH_STATUS_SUCCESS;
835 void *local = talloc_new (NULL);
836 notmuch_database_t *notmuch = NULL;
837 char *notmuch_path, *xapian_path, *incompat_features;
838 char *message = NULL;
841 unsigned int i, version;
842 static int initialized = 0;
845 message = strdup ("Error: Cannot open a database for a NULL path.\n");
846 status = NOTMUCH_STATUS_NULL_POINTER;
850 if (path[0] != '/') {
851 message = strdup ("Error: Database path must be absolute.\n");
852 status = NOTMUCH_STATUS_PATH_ERROR;
856 if (! (notmuch_path = talloc_asprintf (local, "%s/%s", path, ".notmuch"))) {
857 message = strdup ("Out of memory\n");
858 status = NOTMUCH_STATUS_OUT_OF_MEMORY;
862 err = stat (notmuch_path, &st);
864 IGNORE_RESULT (asprintf (&message, "Error opening database at %s: %s\n",
865 notmuch_path, strerror (errno)));
866 status = NOTMUCH_STATUS_FILE_ERROR;
870 if (! (xapian_path = talloc_asprintf (local, "%s/%s", notmuch_path, "xapian"))) {
871 message = strdup ("Out of memory\n");
872 status = NOTMUCH_STATUS_OUT_OF_MEMORY;
876 /* Initialize the GLib type system and threads */
877 #if !GLIB_CHECK_VERSION(2, 35, 1)
881 /* Initialize gmime */
887 notmuch = talloc_zero (NULL, notmuch_database_t);
888 notmuch->exception_reported = false;
889 notmuch->status_string = NULL;
890 notmuch->path = talloc_strdup (notmuch, path);
892 strip_trailing(notmuch->path, '/');
894 notmuch->mode = mode;
895 notmuch->atomic_nesting = 0;
898 string last_thread_id;
901 if (mode == NOTMUCH_DATABASE_MODE_READ_WRITE) {
902 notmuch->xapian_db = new Xapian::WritableDatabase (xapian_path,
905 notmuch->xapian_db = new Xapian::Database (xapian_path);
908 /* Check version. As of database version 3, we represent
909 * changes in terms of features, so assume a version bump
910 * means a dramatically incompatible change. */
911 version = notmuch_database_get_version (notmuch);
912 if (version > NOTMUCH_DATABASE_VERSION) {
913 IGNORE_RESULT (asprintf (&message,
914 "Error: Notmuch database at %s\n"
915 " has a newer database format version (%u) than supported by this\n"
916 " version of notmuch (%u).\n",
917 notmuch_path, version, NOTMUCH_DATABASE_VERSION));
918 notmuch->mode = NOTMUCH_DATABASE_MODE_READ_ONLY;
919 notmuch_database_destroy (notmuch);
921 status = NOTMUCH_STATUS_FILE_ERROR;
925 /* Check features. */
926 incompat_features = NULL;
927 notmuch->features = _parse_features (
928 local, notmuch->xapian_db->get_metadata ("features").c_str (),
929 version, mode == NOTMUCH_DATABASE_MODE_READ_WRITE ? 'w' : 'r',
931 if (incompat_features) {
932 IGNORE_RESULT (asprintf (&message,
933 "Error: Notmuch database at %s\n"
934 " requires features (%s)\n"
935 " not supported by this version of notmuch.\n",
936 notmuch_path, incompat_features));
937 notmuch->mode = NOTMUCH_DATABASE_MODE_READ_ONLY;
938 notmuch_database_destroy (notmuch);
940 status = NOTMUCH_STATUS_FILE_ERROR;
944 notmuch->last_doc_id = notmuch->xapian_db->get_lastdocid ();
945 last_thread_id = notmuch->xapian_db->get_metadata ("last_thread_id");
946 if (last_thread_id.empty ()) {
947 notmuch->last_thread_id = 0;
952 str = last_thread_id.c_str ();
953 notmuch->last_thread_id = strtoull (str, &end, 16);
955 INTERNAL_ERROR ("Malformed database last_thread_id: %s", str);
958 /* Get current highest revision number. */
959 last_mod = notmuch->xapian_db->get_value_upper_bound (
960 NOTMUCH_VALUE_LAST_MOD);
961 if (last_mod.empty ())
962 notmuch->revision = 0;
964 notmuch->revision = Xapian::sortable_unserialise (last_mod);
965 notmuch->uuid = talloc_strdup (
966 notmuch, notmuch->xapian_db->get_uuid ().c_str ());
968 notmuch->query_parser = new Xapian::QueryParser;
969 notmuch->term_gen = new Xapian::TermGenerator;
970 notmuch->term_gen->set_stemmer (Xapian::Stem ("english"));
971 notmuch->value_range_processor = new Xapian::NumberValueRangeProcessor (NOTMUCH_VALUE_TIMESTAMP);
972 notmuch->date_range_processor = new ParseTimeValueRangeProcessor (NOTMUCH_VALUE_TIMESTAMP);
973 notmuch->last_mod_range_processor = new Xapian::NumberValueRangeProcessor (NOTMUCH_VALUE_LAST_MOD, "lastmod:");
975 notmuch->query_parser->set_default_op (Xapian::Query::OP_AND);
976 notmuch->query_parser->set_database (*notmuch->xapian_db);
977 notmuch->query_parser->set_stemmer (Xapian::Stem ("english"));
978 notmuch->query_parser->set_stemming_strategy (Xapian::QueryParser::STEM_SOME);
979 notmuch->query_parser->add_valuerangeprocessor (notmuch->value_range_processor);
980 notmuch->query_parser->add_valuerangeprocessor (notmuch->date_range_processor);
981 notmuch->query_parser->add_valuerangeprocessor (notmuch->last_mod_range_processor);
983 for (i = 0; i < ARRAY_SIZE (prefix_table); i++) {
984 const prefix_t *prefix = &prefix_table[i];
985 if (prefix->flags & NOTMUCH_FIELD_EXTERNAL) {
986 _setup_query_field (prefix, notmuch);
989 } catch (const Xapian::Error &error) {
990 IGNORE_RESULT (asprintf (&message, "A Xapian exception occurred opening database: %s\n",
991 error.get_msg().c_str()));
992 notmuch_database_destroy (notmuch);
994 status = NOTMUCH_STATUS_XAPIAN_EXCEPTION;
1002 *status_string = message;
1008 *database = notmuch;
1010 talloc_free (notmuch);
1015 notmuch_database_close (notmuch_database_t *notmuch)
1017 notmuch_status_t status = NOTMUCH_STATUS_SUCCESS;
1019 /* Many Xapian objects (and thus notmuch objects) hold references to
1020 * the database, so merely deleting the database may not suffice to
1021 * close it. Thus, we explicitly close it here. */
1022 if (notmuch->xapian_db != NULL) {
1024 /* If there's an outstanding transaction, it's unclear if
1025 * closing the Xapian database commits everything up to
1026 * that transaction, or may discard committed (but
1027 * unflushed) transactions. To be certain, explicitly
1028 * cancel any outstanding transaction before closing. */
1029 if (notmuch->mode == NOTMUCH_DATABASE_MODE_READ_WRITE &&
1030 notmuch->atomic_nesting)
1031 (static_cast <Xapian::WritableDatabase *> (notmuch->xapian_db))
1032 ->cancel_transaction ();
1034 /* Close the database. This implicitly flushes
1035 * outstanding changes. */
1036 notmuch->xapian_db->close();
1037 } catch (const Xapian::Error &error) {
1038 status = NOTMUCH_STATUS_XAPIAN_EXCEPTION;
1039 if (! notmuch->exception_reported) {
1040 _notmuch_database_log (notmuch, "Error: A Xapian exception occurred closing database: %s\n",
1041 error.get_msg().c_str());
1046 delete notmuch->term_gen;
1047 notmuch->term_gen = NULL;
1048 delete notmuch->query_parser;
1049 notmuch->query_parser = NULL;
1050 delete notmuch->xapian_db;
1051 notmuch->xapian_db = NULL;
1052 delete notmuch->value_range_processor;
1053 notmuch->value_range_processor = NULL;
1054 delete notmuch->date_range_processor;
1055 notmuch->date_range_processor = NULL;
1056 delete notmuch->last_mod_range_processor;
1057 notmuch->last_mod_range_processor = NULL;
1063 _notmuch_database_reopen (notmuch_database_t *notmuch)
1065 if (notmuch->mode != NOTMUCH_DATABASE_MODE_READ_ONLY)
1066 return NOTMUCH_STATUS_UNSUPPORTED_OPERATION;
1069 notmuch->xapian_db->reopen ();
1070 } catch (const Xapian::Error &error) {
1071 if (! notmuch->exception_reported) {
1072 _notmuch_database_log (notmuch, "Error: A Xapian exception reopening database: %s\n",
1073 error.get_msg ().c_str ());
1074 notmuch->exception_reported = true;
1076 return NOTMUCH_STATUS_XAPIAN_EXCEPTION;
1081 return NOTMUCH_STATUS_SUCCESS;
1085 unlink_cb (const char *path,
1086 unused (const struct stat *sb),
1088 unused (struct FTW *ftw))
1090 return remove (path);
1094 rmtree (const char *path)
1096 return nftw (path, unlink_cb, 64, FTW_DEPTH | FTW_PHYS);
1099 class NotmuchCompactor : public Xapian::Compactor
1101 notmuch_compact_status_cb_t status_cb;
1102 void *status_closure;
1105 NotmuchCompactor(notmuch_compact_status_cb_t cb, void *closure) :
1106 status_cb (cb), status_closure (closure) { }
1109 set_status (const std::string &table, const std::string &status)
1113 if (status_cb == NULL)
1116 if (status.length () == 0)
1117 msg = talloc_asprintf (NULL, "compacting table %s", table.c_str());
1119 msg = talloc_asprintf (NULL, " %s", status.c_str());
1125 status_cb (msg, status_closure);
1130 /* Compacts the given database, optionally saving the original database
1131 * in backup_path. Additionally, a callback function can be provided to
1132 * give the user feedback on the progress of the (likely long-lived)
1133 * compaction process.
1135 * The backup path must point to a directory on the same volume as the
1136 * original database. Passing a NULL backup_path will result in the
1137 * uncompacted database being deleted after compaction has finished.
1138 * Note that the database write lock will be held during the
1139 * compaction process to protect data integrity.
1142 notmuch_database_compact (const char *path,
1143 const char *backup_path,
1144 notmuch_compact_status_cb_t status_cb,
1148 char *notmuch_path, *xapian_path, *compact_xapian_path;
1149 notmuch_status_t ret = NOTMUCH_STATUS_SUCCESS;
1150 notmuch_database_t *notmuch = NULL;
1151 struct stat statbuf;
1153 char *message = NULL;
1155 local = talloc_new (NULL);
1157 return NOTMUCH_STATUS_OUT_OF_MEMORY;
1159 ret = notmuch_database_open_verbose (path,
1160 NOTMUCH_DATABASE_MODE_READ_WRITE,
1164 if (status_cb) status_cb (message, closure);
1168 if (! (notmuch_path = talloc_asprintf (local, "%s/%s", path, ".notmuch"))) {
1169 ret = NOTMUCH_STATUS_OUT_OF_MEMORY;
1173 if (! (xapian_path = talloc_asprintf (local, "%s/%s", notmuch_path, "xapian"))) {
1174 ret = NOTMUCH_STATUS_OUT_OF_MEMORY;
1178 if (! (compact_xapian_path = talloc_asprintf (local, "%s.compact", xapian_path))) {
1179 ret = NOTMUCH_STATUS_OUT_OF_MEMORY;
1183 if (backup_path == NULL) {
1184 if (! (backup_path = talloc_asprintf (local, "%s.old", xapian_path))) {
1185 ret = NOTMUCH_STATUS_OUT_OF_MEMORY;
1188 keep_backup = false;
1194 if (stat (backup_path, &statbuf) != -1) {
1195 _notmuch_database_log (notmuch, "Path already exists: %s\n", backup_path);
1196 ret = NOTMUCH_STATUS_FILE_ERROR;
1199 if (errno != ENOENT) {
1200 _notmuch_database_log (notmuch, "Unknown error while stat()ing path: %s\n",
1202 ret = NOTMUCH_STATUS_FILE_ERROR;
1206 /* Unconditionally attempt to remove old work-in-progress database (if
1207 * any). This is "protected" by database lock. If this fails due to write
1208 * errors (etc), the following code will fail and provide error message.
1210 (void) rmtree (compact_xapian_path);
1213 NotmuchCompactor compactor (status_cb, closure);
1215 compactor.set_renumber (false);
1216 compactor.add_source (xapian_path);
1217 compactor.set_destdir (compact_xapian_path);
1218 compactor.compact ();
1219 } catch (const Xapian::Error &error) {
1220 _notmuch_database_log (notmuch, "Error while compacting: %s\n", error.get_msg().c_str());
1221 ret = NOTMUCH_STATUS_XAPIAN_EXCEPTION;
1225 if (rename (xapian_path, backup_path)) {
1226 _notmuch_database_log (notmuch, "Error moving %s to %s: %s\n",
1227 xapian_path, backup_path, strerror (errno));
1228 ret = NOTMUCH_STATUS_FILE_ERROR;
1232 if (rename (compact_xapian_path, xapian_path)) {
1233 _notmuch_database_log (notmuch, "Error moving %s to %s: %s\n",
1234 compact_xapian_path, xapian_path, strerror (errno));
1235 ret = NOTMUCH_STATUS_FILE_ERROR;
1239 if (! keep_backup) {
1240 if (rmtree (backup_path)) {
1241 _notmuch_database_log (notmuch, "Error removing old database %s: %s\n",
1242 backup_path, strerror (errno));
1243 ret = NOTMUCH_STATUS_FILE_ERROR;
1250 notmuch_status_t ret2;
1252 const char *str = notmuch_database_status_string (notmuch);
1253 if (status_cb && str)
1254 status_cb (str, closure);
1256 ret2 = notmuch_database_destroy (notmuch);
1258 /* don't clobber previous error status */
1259 if (ret == NOTMUCH_STATUS_SUCCESS && ret2 != NOTMUCH_STATUS_SUCCESS)
1263 talloc_free (local);
1269 notmuch_database_destroy (notmuch_database_t *notmuch)
1271 notmuch_status_t status;
1273 status = notmuch_database_close (notmuch);
1274 talloc_free (notmuch);
1280 notmuch_database_get_path (notmuch_database_t *notmuch)
1282 return notmuch->path;
1286 notmuch_database_get_version (notmuch_database_t *notmuch)
1288 unsigned int version;
1289 string version_string;
1293 version_string = notmuch->xapian_db->get_metadata ("version");
1294 if (version_string.empty ())
1297 str = version_string.c_str ();
1298 if (str == NULL || *str == '\0')
1301 version = strtoul (str, &end, 10);
1303 INTERNAL_ERROR ("Malformed database version: %s", str);
1309 notmuch_database_needs_upgrade (notmuch_database_t *notmuch)
1311 return notmuch->mode == NOTMUCH_DATABASE_MODE_READ_WRITE &&
1312 ((NOTMUCH_FEATURES_CURRENT & ~notmuch->features) ||
1313 (notmuch_database_get_version (notmuch) < NOTMUCH_DATABASE_VERSION));
1316 static volatile sig_atomic_t do_progress_notify = 0;
1319 handle_sigalrm (unused (int signal))
1321 do_progress_notify = 1;
1324 /* Upgrade the current database.
1326 * After opening a database in read-write mode, the client should
1327 * check if an upgrade is needed (notmuch_database_needs_upgrade) and
1328 * if so, upgrade with this function before making any modifications.
1330 * The optional progress_notify callback can be used by the caller to
1331 * provide progress indication to the user. If non-NULL it will be
1332 * called periodically with 'count' as the number of messages upgraded
1333 * so far and 'total' the overall number of messages that will be
1337 notmuch_database_upgrade (notmuch_database_t *notmuch,
1338 void (*progress_notify) (void *closure,
1342 void *local = talloc_new (NULL);
1343 Xapian::TermIterator t, t_end;
1344 Xapian::WritableDatabase *db;
1345 struct sigaction action;
1346 struct itimerval timerval;
1347 bool timer_is_active = false;
1348 enum _notmuch_features target_features, new_features;
1349 notmuch_status_t status;
1350 notmuch_private_status_t private_status;
1351 notmuch_query_t *query = NULL;
1352 unsigned int count = 0, total = 0;
1354 status = _notmuch_database_ensure_writable (notmuch);
1358 db = static_cast <Xapian::WritableDatabase *> (notmuch->xapian_db);
1360 target_features = notmuch->features | NOTMUCH_FEATURES_CURRENT;
1361 new_features = NOTMUCH_FEATURES_CURRENT & ~notmuch->features;
1363 if (! notmuch_database_needs_upgrade (notmuch))
1364 return NOTMUCH_STATUS_SUCCESS;
1366 if (progress_notify) {
1367 /* Set up our handler for SIGALRM */
1368 memset (&action, 0, sizeof (struct sigaction));
1369 action.sa_handler = handle_sigalrm;
1370 sigemptyset (&action.sa_mask);
1371 action.sa_flags = SA_RESTART;
1372 sigaction (SIGALRM, &action, NULL);
1374 /* Then start a timer to send SIGALRM once per second. */
1375 timerval.it_interval.tv_sec = 1;
1376 timerval.it_interval.tv_usec = 0;
1377 timerval.it_value.tv_sec = 1;
1378 timerval.it_value.tv_usec = 0;
1379 setitimer (ITIMER_REAL, &timerval, NULL);
1381 timer_is_active = true;
1384 /* Figure out how much total work we need to do. */
1386 (NOTMUCH_FEATURE_FILE_TERMS | NOTMUCH_FEATURE_BOOL_FOLDER |
1387 NOTMUCH_FEATURE_LAST_MOD)) {
1388 query = notmuch_query_create (notmuch, "");
1391 status = notmuch_query_count_messages (query, &msg_count);
1396 notmuch_query_destroy (query);
1399 if (new_features & NOTMUCH_FEATURE_DIRECTORY_DOCS) {
1400 t_end = db->allterms_end ("XTIMESTAMP");
1401 for (t = db->allterms_begin ("XTIMESTAMP"); t != t_end; t++)
1404 if (new_features & NOTMUCH_FEATURE_GHOSTS) {
1405 /* The ghost message upgrade converts all thread_id_*
1406 * metadata values into ghost message documents. */
1407 t_end = db->metadata_keys_end ("thread_id_");
1408 for (t = db->metadata_keys_begin ("thread_id_"); t != t_end; ++t)
1412 /* Perform the upgrade in a transaction. */
1413 db->begin_transaction (true);
1415 /* Set the target features so we write out changes in the desired
1417 notmuch->features = target_features;
1419 /* Perform per-message upgrades. */
1421 (NOTMUCH_FEATURE_FILE_TERMS | NOTMUCH_FEATURE_BOOL_FOLDER |
1422 NOTMUCH_FEATURE_LAST_MOD)) {
1423 notmuch_messages_t *messages;
1424 notmuch_message_t *message;
1427 query = notmuch_query_create (notmuch, "");
1429 status = notmuch_query_search_messages (query, &messages);
1433 notmuch_messages_valid (messages);
1434 notmuch_messages_move_to_next (messages))
1436 if (do_progress_notify) {
1437 progress_notify (closure, (double) count / total);
1438 do_progress_notify = 0;
1441 message = notmuch_messages_get (messages);
1443 /* Before version 1, each message document had its
1444 * filename in the data field. Copy that into the new
1445 * format by calling notmuch_message_add_filename.
1447 if (new_features & NOTMUCH_FEATURE_FILE_TERMS) {
1448 filename = _notmuch_message_talloc_copy_data (message);
1449 if (filename && *filename != '\0') {
1450 _notmuch_message_add_filename (message, filename);
1451 _notmuch_message_clear_data (message);
1453 talloc_free (filename);
1456 /* Prior to version 2, the "folder:" prefix was
1457 * probabilistic and stemmed. Change it to the current
1458 * boolean prefix. Add "path:" prefixes while at it.
1460 if (new_features & NOTMUCH_FEATURE_BOOL_FOLDER)
1461 _notmuch_message_upgrade_folder (message);
1463 /* Prior to NOTMUCH_FEATURE_LAST_MOD, messages did not
1464 * track modification revisions. Give all messages the
1465 * next available revision; since we just started tracking
1466 * revisions for this database, that will be 1.
1468 if (new_features & NOTMUCH_FEATURE_LAST_MOD)
1469 _notmuch_message_upgrade_last_mod (message);
1471 _notmuch_message_sync (message);
1473 notmuch_message_destroy (message);
1478 notmuch_query_destroy (query);
1482 /* Perform per-directory upgrades. */
1484 /* Before version 1 we stored directory timestamps in
1485 * XTIMESTAMP documents instead of the current XDIRECTORY
1486 * documents. So copy those as well. */
1487 if (new_features & NOTMUCH_FEATURE_DIRECTORY_DOCS) {
1488 t_end = notmuch->xapian_db->allterms_end ("XTIMESTAMP");
1490 for (t = notmuch->xapian_db->allterms_begin ("XTIMESTAMP");
1494 Xapian::PostingIterator p, p_end;
1495 std::string term = *t;
1497 p_end = notmuch->xapian_db->postlist_end (term);
1499 for (p = notmuch->xapian_db->postlist_begin (term);
1503 Xapian::Document document;
1505 notmuch_directory_t *directory;
1507 if (do_progress_notify) {
1508 progress_notify (closure, (double) count / total);
1509 do_progress_notify = 0;
1512 document = find_document_for_doc_id (notmuch, *p);
1513 mtime = Xapian::sortable_unserialise (
1514 document.get_value (NOTMUCH_VALUE_TIMESTAMP));
1516 directory = _notmuch_directory_create (notmuch, term.c_str() + 10,
1517 NOTMUCH_FIND_CREATE, &status);
1518 notmuch_directory_set_mtime (directory, mtime);
1519 notmuch_directory_destroy (directory);
1521 db->delete_document (*p);
1528 /* Perform metadata upgrades. */
1530 /* Prior to NOTMUCH_FEATURE_GHOSTS, thread IDs for missing
1531 * messages were stored as database metadata. Change these to
1534 if (new_features & NOTMUCH_FEATURE_GHOSTS) {
1535 notmuch_message_t *message;
1536 std::string message_id, thread_id;
1538 t_end = db->metadata_keys_end (NOTMUCH_METADATA_THREAD_ID_PREFIX);
1539 for (t = db->metadata_keys_begin (NOTMUCH_METADATA_THREAD_ID_PREFIX);
1541 if (do_progress_notify) {
1542 progress_notify (closure, (double) count / total);
1543 do_progress_notify = 0;
1546 message_id = (*t).substr (
1547 strlen (NOTMUCH_METADATA_THREAD_ID_PREFIX));
1548 thread_id = db->get_metadata (*t);
1550 /* Create ghost message */
1551 message = _notmuch_message_create_for_message_id (
1552 notmuch, message_id.c_str (), &private_status);
1553 if (private_status == NOTMUCH_PRIVATE_STATUS_SUCCESS) {
1554 /* Document already exists; ignore the stored thread ID */
1555 } else if (private_status ==
1556 NOTMUCH_PRIVATE_STATUS_NO_DOCUMENT_FOUND) {
1557 private_status = _notmuch_message_initialize_ghost (
1558 message, thread_id.c_str ());
1559 if (! private_status)
1560 _notmuch_message_sync (message);
1563 if (private_status) {
1564 _notmuch_database_log (notmuch,
1565 "Upgrade failed while creating ghost messages.\n");
1566 status = COERCE_STATUS (private_status, "Unexpected status from _notmuch_message_initialize_ghost");
1570 /* Clear saved metadata thread ID */
1571 db->set_metadata (*t, "");
1577 status = NOTMUCH_STATUS_SUCCESS;
1578 db->set_metadata ("features", _print_features (local, notmuch->features));
1579 db->set_metadata ("version", STRINGIFY (NOTMUCH_DATABASE_VERSION));
1582 if (status == NOTMUCH_STATUS_SUCCESS)
1583 db->commit_transaction ();
1585 db->cancel_transaction ();
1587 if (timer_is_active) {
1588 /* Now stop the timer. */
1589 timerval.it_interval.tv_sec = 0;
1590 timerval.it_interval.tv_usec = 0;
1591 timerval.it_value.tv_sec = 0;
1592 timerval.it_value.tv_usec = 0;
1593 setitimer (ITIMER_REAL, &timerval, NULL);
1595 /* And disable the signal handler. */
1596 action.sa_handler = SIG_IGN;
1597 sigaction (SIGALRM, &action, NULL);
1601 notmuch_query_destroy (query);
1603 talloc_free (local);
1608 notmuch_database_begin_atomic (notmuch_database_t *notmuch)
1610 if (notmuch->mode == NOTMUCH_DATABASE_MODE_READ_ONLY ||
1611 notmuch->atomic_nesting > 0)
1614 if (notmuch_database_needs_upgrade (notmuch))
1615 return NOTMUCH_STATUS_UPGRADE_REQUIRED;
1618 (static_cast <Xapian::WritableDatabase *> (notmuch->xapian_db))->begin_transaction (false);
1619 } catch (const Xapian::Error &error) {
1620 _notmuch_database_log (notmuch, "A Xapian exception occurred beginning transaction: %s.\n",
1621 error.get_msg().c_str());
1622 notmuch->exception_reported = true;
1623 return NOTMUCH_STATUS_XAPIAN_EXCEPTION;
1627 notmuch->atomic_nesting++;
1628 return NOTMUCH_STATUS_SUCCESS;
1632 notmuch_database_end_atomic (notmuch_database_t *notmuch)
1634 Xapian::WritableDatabase *db;
1636 if (notmuch->atomic_nesting == 0)
1637 return NOTMUCH_STATUS_UNBALANCED_ATOMIC;
1639 if (notmuch->mode == NOTMUCH_DATABASE_MODE_READ_ONLY ||
1640 notmuch->atomic_nesting > 1)
1643 db = static_cast <Xapian::WritableDatabase *> (notmuch->xapian_db);
1645 db->commit_transaction ();
1647 /* This is a hack for testing. Xapian never flushes on a
1648 * non-flushed commit, even if the flush threshold is 1.
1649 * However, we rely on flushing to test atomicity. */
1650 const char *thresh = getenv ("XAPIAN_FLUSH_THRESHOLD");
1651 if (thresh && atoi (thresh) == 1)
1653 } catch (const Xapian::Error &error) {
1654 _notmuch_database_log (notmuch, "A Xapian exception occurred committing transaction: %s.\n",
1655 error.get_msg().c_str());
1656 notmuch->exception_reported = true;
1657 return NOTMUCH_STATUS_XAPIAN_EXCEPTION;
1660 if (notmuch->atomic_dirty) {
1661 ++notmuch->revision;
1662 notmuch->atomic_dirty = false;
1666 notmuch->atomic_nesting--;
1667 return NOTMUCH_STATUS_SUCCESS;
1671 notmuch_database_get_revision (notmuch_database_t *notmuch,
1675 *uuid = notmuch->uuid;
1676 return notmuch->revision;
1679 /* We allow the user to use arbitrarily long paths for directories. But
1680 * we have a term-length limit. So if we exceed that, we'll use the
1681 * SHA-1 of the path for the database term.
1683 * Note: This function may return the original value of 'path'. If it
1684 * does not, then the caller is responsible to free() the returned
1688 _notmuch_database_get_directory_db_path (const char *path)
1690 int term_len = strlen (_find_prefix ("directory")) + strlen (path);
1692 if (term_len > NOTMUCH_TERM_MAX)
1693 return _notmuch_sha1_of_string (path);
1698 /* Given a path, split it into two parts: the directory part is all
1699 * components except for the last, and the basename is that last
1700 * component. Getting the return-value for either part is optional
1701 * (the caller can pass NULL).
1703 * The original 'path' can represent either a regular file or a
1704 * directory---the splitting will be carried out in the same way in
1705 * either case. Trailing slashes on 'path' will be ignored, and any
1706 * cases of multiple '/' characters appearing in series will be
1707 * treated as a single '/'.
1709 * Allocation (if any) will have 'ctx' as the talloc owner. But
1710 * pointers will be returned within the original path string whenever
1713 * Note: If 'path' is non-empty and contains no non-trailing slash,
1714 * (that is, consists of a filename with no parent directory), then
1715 * the directory returned will be an empty string. However, if 'path'
1716 * is an empty string, then both directory and basename will be
1720 _notmuch_database_split_path (void *ctx,
1722 const char **directory,
1723 const char **basename)
1727 if (path == NULL || *path == '\0') {
1732 return NOTMUCH_STATUS_SUCCESS;
1735 /* Find the last slash (not counting a trailing slash), if any. */
1737 slash = path + strlen (path) - 1;
1739 /* First, skip trailing slashes. */
1740 while (slash != path && *slash == '/')
1743 /* Then, find a slash. */
1744 while (slash != path && *slash != '/') {
1751 /* Finally, skip multiple slashes. */
1752 while (slash != path && *(slash - 1) == '/')
1755 if (slash == path) {
1757 *directory = talloc_strdup (ctx, "");
1762 *directory = talloc_strndup (ctx, path, slash - path);
1765 return NOTMUCH_STATUS_SUCCESS;
1768 /* Find the document ID of the specified directory.
1770 * If (flags & NOTMUCH_FIND_CREATE), a new directory document will be
1771 * created if one does not exist for 'path'. Otherwise, if the
1772 * directory document does not exist, this sets *directory_id to
1773 * ((unsigned int)-1) and returns NOTMUCH_STATUS_SUCCESS.
1776 _notmuch_database_find_directory_id (notmuch_database_t *notmuch,
1778 notmuch_find_flags_t flags,
1779 unsigned int *directory_id)
1781 notmuch_directory_t *directory;
1782 notmuch_status_t status;
1786 return NOTMUCH_STATUS_SUCCESS;
1789 directory = _notmuch_directory_create (notmuch, path, flags, &status);
1790 if (status || !directory) {
1795 *directory_id = _notmuch_directory_get_document_id (directory);
1797 notmuch_directory_destroy (directory);
1799 return NOTMUCH_STATUS_SUCCESS;
1803 _notmuch_database_get_directory_path (void *ctx,
1804 notmuch_database_t *notmuch,
1805 unsigned int doc_id)
1807 Xapian::Document document;
1809 document = find_document_for_doc_id (notmuch, doc_id);
1811 return talloc_strdup (ctx, document.get_data ().c_str ());
1814 /* Given a legal 'filename' for the database, (either relative to
1815 * database path or absolute with initial components identical to
1816 * database path), return a new string (with 'ctx' as the talloc
1817 * owner) suitable for use as a direntry term value.
1819 * If (flags & NOTMUCH_FIND_CREATE), the necessary directory documents
1820 * will be created in the database as needed. Otherwise, if the
1821 * necessary directory documents do not exist, this sets
1822 * *direntry to NULL and returns NOTMUCH_STATUS_SUCCESS.
1825 _notmuch_database_filename_to_direntry (void *ctx,
1826 notmuch_database_t *notmuch,
1827 const char *filename,
1828 notmuch_find_flags_t flags,
1831 const char *relative, *directory, *basename;
1832 Xapian::docid directory_id;
1833 notmuch_status_t status;
1835 relative = _notmuch_database_relative_path (notmuch, filename);
1837 status = _notmuch_database_split_path (ctx, relative,
1838 &directory, &basename);
1842 status = _notmuch_database_find_directory_id (notmuch, directory, flags,
1844 if (status || directory_id == (unsigned int)-1) {
1849 *direntry = talloc_asprintf (ctx, "%u:%s", directory_id, basename);
1851 return NOTMUCH_STATUS_SUCCESS;
1854 /* Given a legal 'path' for the database, return the relative path.
1856 * The return value will be a pointer to the original path contents,
1857 * and will be either the original string (if 'path' was relative) or
1858 * a portion of the string (if path was absolute and begins with the
1862 _notmuch_database_relative_path (notmuch_database_t *notmuch,
1865 const char *db_path, *relative;
1866 unsigned int db_path_len;
1868 db_path = notmuch_database_get_path (notmuch);
1869 db_path_len = strlen (db_path);
1873 if (*relative == '/') {
1874 while (*relative == '/' && *(relative+1) == '/')
1877 if (strncmp (relative, db_path, db_path_len) == 0)
1879 relative += db_path_len;
1880 while (*relative == '/')
1889 notmuch_database_get_directory (notmuch_database_t *notmuch,
1891 notmuch_directory_t **directory)
1893 notmuch_status_t status;
1895 if (directory == NULL)
1896 return NOTMUCH_STATUS_NULL_POINTER;
1900 *directory = _notmuch_directory_create (notmuch, path,
1901 NOTMUCH_FIND_LOOKUP, &status);
1902 } catch (const Xapian::Error &error) {
1903 _notmuch_database_log (notmuch, "A Xapian exception occurred getting directory: %s.\n",
1904 error.get_msg().c_str());
1905 notmuch->exception_reported = true;
1906 status = NOTMUCH_STATUS_XAPIAN_EXCEPTION;
1911 /* Allocate a document ID that satisfies the following criteria:
1913 * 1. The ID does not exist for any document in the Xapian database
1915 * 2. The ID was not previously returned from this function
1917 * 3. The ID is the smallest integer satisfying (1) and (2)
1919 * This function will trigger an internal error if these constraints
1920 * cannot all be satisfied, (that is, the pool of available document
1921 * IDs has been exhausted).
1924 _notmuch_database_generate_doc_id (notmuch_database_t *notmuch)
1926 assert (notmuch->last_doc_id >= notmuch->xapian_db->get_lastdocid ());
1928 notmuch->last_doc_id++;
1930 if (notmuch->last_doc_id == 0)
1931 INTERNAL_ERROR ("Xapian document IDs are exhausted.\n");
1933 return notmuch->last_doc_id;
1937 notmuch_database_remove_message (notmuch_database_t *notmuch,
1938 const char *filename)
1940 notmuch_status_t status;
1941 notmuch_message_t *message;
1943 status = notmuch_database_find_message_by_filename (notmuch, filename,
1946 if (status == NOTMUCH_STATUS_SUCCESS && message) {
1947 status = _notmuch_message_remove_filename (message, filename);
1948 if (status == NOTMUCH_STATUS_SUCCESS)
1949 _notmuch_message_delete (message);
1950 else if (status == NOTMUCH_STATUS_DUPLICATE_MESSAGE_ID)
1951 _notmuch_message_sync (message);
1953 notmuch_message_destroy (message);
1960 notmuch_database_find_message_by_filename (notmuch_database_t *notmuch,
1961 const char *filename,
1962 notmuch_message_t **message_ret)
1965 const char *prefix = _find_prefix ("file-direntry");
1966 char *direntry, *term;
1967 Xapian::PostingIterator i, end;
1968 notmuch_status_t status;
1970 if (message_ret == NULL)
1971 return NOTMUCH_STATUS_NULL_POINTER;
1973 if (! (notmuch->features & NOTMUCH_FEATURE_FILE_TERMS))
1974 return NOTMUCH_STATUS_UPGRADE_REQUIRED;
1976 /* return NULL on any failure */
1977 *message_ret = NULL;
1979 local = talloc_new (notmuch);
1982 status = _notmuch_database_filename_to_direntry (
1983 local, notmuch, filename, NOTMUCH_FIND_LOOKUP, &direntry);
1984 if (status || !direntry)
1987 term = talloc_asprintf (local, "%s%s", prefix, direntry);
1989 find_doc_ids_for_term (notmuch, term, &i, &end);
1992 notmuch_private_status_t private_status;
1994 *message_ret = _notmuch_message_create (notmuch, notmuch, *i,
1996 if (*message_ret == NULL)
1997 status = NOTMUCH_STATUS_OUT_OF_MEMORY;
1999 } catch (const Xapian::Error &error) {
2000 _notmuch_database_log (notmuch, "Error: A Xapian exception occurred finding message by filename: %s\n",
2001 error.get_msg().c_str());
2002 notmuch->exception_reported = true;
2003 status = NOTMUCH_STATUS_XAPIAN_EXCEPTION;
2007 talloc_free (local);
2009 if (status && *message_ret) {
2010 notmuch_message_destroy (*message_ret);
2011 *message_ret = NULL;
2016 notmuch_string_list_t *
2017 _notmuch_database_get_terms_with_prefix (void *ctx, Xapian::TermIterator &i,
2018 Xapian::TermIterator &end,
2021 int prefix_len = strlen (prefix);
2022 notmuch_string_list_t *list;
2024 list = _notmuch_string_list_create (ctx);
2025 if (unlikely (list == NULL))
2028 for (i.skip_to (prefix); i != end; i++) {
2029 /* Terminate loop at first term without desired prefix. */
2030 if (strncmp ((*i).c_str (), prefix, prefix_len))
2033 _notmuch_string_list_append (list, (*i).c_str () + prefix_len);
2040 notmuch_database_get_all_tags (notmuch_database_t *db)
2042 Xapian::TermIterator i, end;
2043 notmuch_string_list_t *tags;
2046 i = db->xapian_db->allterms_begin();
2047 end = db->xapian_db->allterms_end();
2048 tags = _notmuch_database_get_terms_with_prefix (db, i, end,
2049 _find_prefix ("tag"));
2050 _notmuch_string_list_sort (tags);
2051 return _notmuch_tags_create (db, tags);
2052 } catch (const Xapian::Error &error) {
2053 _notmuch_database_log (db, "A Xapian exception occurred getting tags: %s.\n",
2054 error.get_msg().c_str());
2055 db->exception_reported = true;
2061 notmuch_database_status_string (const notmuch_database_t *notmuch)
2063 return notmuch->status_string;