1 /* database.cc - The database interfaces of the notmuch mail library
3 * Copyright © 2009 Carl Worth
5 * This program is free software: you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation, either version 3 of the License, or
8 * (at your option) any later version.
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
15 * You should have received a copy of the GNU General Public License
16 * along with this program. If not, see http://www.gnu.org/licenses/ .
18 * Author: Carl Worth <cworth@cworth.org>
21 #include "database-private.h"
27 #include <glib.h> /* g_strdup_printf, g_free, GPtrArray, GHashTable */
31 #define ARRAY_SIZE(arr) (sizeof (arr) / sizeof (arr[0]))
33 /* These prefix values are specifically chosen to be compatible
34 * with sup, (http://sup.rubyforge.org), written by
35 * William Morgan <wmorgan-sup@masanjin.net>, and released
36 * under the GNU GPL v2.
44 prefix_t BOOLEAN_PREFIX[] = {
50 { "timestamp", "KTS" },
54 _find_prefix (const char *name)
58 for (i = 0; i < ARRAY_SIZE (BOOLEAN_PREFIX); i++)
59 if (strcmp (name, BOOLEAN_PREFIX[i].name) == 0)
60 return BOOLEAN_PREFIX[i].prefix;
62 fprintf (stderr, "Internal error: No prefix exists for '%s'\n", name);
69 notmuch_status_to_string (notmuch_status_t status)
72 case NOTMUCH_STATUS_SUCCESS:
73 return "No error occurred";
74 case NOTMUCH_STATUS_XAPIAN_EXCEPTION:
75 return "A Xapian exception occurred";
76 case NOTMUCH_STATUS_FILE_ERROR:
77 return "Something went wrong trying to read or write a file";
78 case NOTMUCH_STATUS_FILE_NOT_EMAIL:
79 return "File is not an email";
80 case NOTMUCH_STATUS_DUPLICATE_MESSAGE_ID:
81 return "Message ID is identical to a message in database";
82 case NOTMUCH_STATUS_NULL_POINTER:
83 return "Erroneous NULL pointer";
84 case NOTMUCH_STATUS_TAG_TOO_LONG:
85 return "Tag value is too long (exceeds NOTMUCH_TAG_MAX)";
87 case NOTMUCH_STATUS_LAST_STATUS:
88 return "Unknown error status value";
92 /* XXX: We should drop this function and convert all callers to call
93 * _notmuch_message_add_term instead. */
95 add_term (Xapian::Document doc,
96 const char *prefix_name,
105 prefix = _find_prefix (prefix_name);
107 term = g_strdup_printf ("%s%s", prefix, value);
109 if (strlen (term) <= NOTMUCH_TERM_MAX)
116 find_doc_ids (notmuch_database_t *notmuch,
117 const char *prefix_name,
119 Xapian::PostingIterator *begin,
120 Xapian::PostingIterator *end)
122 Xapian::PostingIterator i;
125 term = g_strdup_printf ("%s%s", _find_prefix (prefix_name), value);
127 *begin = notmuch->xapian_db->postlist_begin (term);
129 *end = notmuch->xapian_db->postlist_end (term);
134 static notmuch_private_status_t
135 find_unique_doc_id (notmuch_database_t *notmuch,
136 const char *prefix_name,
138 unsigned int *doc_id)
140 Xapian::PostingIterator i, end;
142 find_doc_ids (notmuch, prefix_name, value, &i, &end);
146 return NOTMUCH_PRIVATE_STATUS_NO_DOCUMENT_FOUND;
149 return NOTMUCH_PRIVATE_STATUS_SUCCESS;
153 static Xapian::Document
154 find_document_for_doc_id (notmuch_database_t *notmuch, unsigned doc_id)
156 return notmuch->xapian_db->get_document (doc_id);
159 static notmuch_private_status_t
160 find_unique_document (notmuch_database_t *notmuch,
161 const char *prefix_name,
163 Xapian::Document *document,
164 unsigned int *doc_id)
166 notmuch_private_status_t status;
168 status = find_unique_doc_id (notmuch, prefix_name, value, doc_id);
171 *document = Xapian::Document ();
175 *document = find_document_for_doc_id (notmuch, *doc_id);
176 return NOTMUCH_PRIVATE_STATUS_SUCCESS;
180 insert_thread_id (GHashTable *thread_ids, Xapian::Document doc)
183 const char *value, *id, *comma;
185 value_string = doc.get_value (NOTMUCH_VALUE_THREAD);
186 value = value_string.c_str();
187 if (strlen (value)) {
190 comma = strchr (id, ',');
192 comma = id + strlen (id);
193 g_hash_table_insert (thread_ids,
194 strndup (id, comma - id), NULL);
203 notmuch_database_find_message (notmuch_database_t *notmuch,
204 const char *message_id)
206 notmuch_private_status_t status;
209 status = find_unique_doc_id (notmuch, "msgid", message_id, &doc_id);
211 if (status == NOTMUCH_PRIVATE_STATUS_NO_DOCUMENT_FOUND)
214 return _notmuch_message_create (notmuch, notmuch, doc_id);
217 /* Return one or more thread_ids, (as a GPtrArray of strings), for the
218 * given message based on looking into the database for any messages
219 * referenced in parents, and also for any messages in the database
220 * referencing message_id.
222 * Caller should free all strings in the array and the array itself,
223 * (g_ptr_array_free) when done. */
225 find_thread_ids (notmuch_database_t *notmuch,
227 const char *message_id)
229 Xapian::PostingIterator child, children_end;
230 Xapian::Document doc;
231 GHashTable *thread_ids;
234 const char *parent_message_id;
237 thread_ids = g_hash_table_new_full (g_str_hash, g_str_equal,
240 find_doc_ids (notmuch, "ref", message_id, &child, &children_end);
241 for ( ; child != children_end; child++) {
242 doc = find_document_for_doc_id (notmuch, *child);
243 insert_thread_id (thread_ids, doc);
246 for (i = 0; i < parents->len; i++) {
247 notmuch_message_t *parent;
248 notmuch_thread_ids_t *ids;
250 parent_message_id = (char *) g_ptr_array_index (parents, i);
251 parent = notmuch_database_find_message (notmuch, parent_message_id);
255 for (ids = notmuch_message_get_thread_ids (parent);
256 notmuch_thread_ids_has_more (ids);
257 notmuch_thread_ids_advance (ids))
261 id = notmuch_thread_ids_get (ids);
262 g_hash_table_insert (thread_ids, strdup (id), NULL);
265 notmuch_message_destroy (parent);
268 result = g_ptr_array_new ();
270 keys = g_hash_table_get_keys (thread_ids);
271 for (l = keys; l; l = l->next) {
272 char *id = (char *) l->data;
273 g_ptr_array_add (result, id);
277 /* We're done with the hash table, but we've taken the pointers to
278 * the allocated strings and put them into our result array, so
279 * tell the hash not to free them on its way out. */
280 g_hash_table_steal_all (thread_ids);
281 g_hash_table_unref (thread_ids);
286 /* Advance 'str' past any whitespace or RFC 822 comments. A comment is
287 * a (potentially nested) parenthesized sequence with '\' used to
288 * escape any character (including parentheses).
290 * If the sequence to be skipped continues to the end of the string,
291 * then 'str' will be left pointing at the final terminating '\0'
295 skip_space_and_comments (const char **str)
300 while (*s && (isspace (*s) || *s == '(')) {
301 while (*s && isspace (*s))
306 while (*s && nesting) {
322 /* Parse an RFC 822 message-id, discarding whitespace, any RFC 822
323 * comments, and the '<' and '>' delimeters.
325 * If not NULL, then *next will be made to point to the first character
326 * not parsed, (possibly pointing to the final '\0' terminator.
328 * Returns a newly allocated string which the caller should free()
331 * Returns NULL if there is any error parsing the message-id. */
333 parse_message_id (const char *message_id, const char **next)
338 if (message_id == NULL)
343 skip_space_and_comments (&s);
345 /* Skip any unstructured text as well. */
346 while (*s && *s != '<')
357 skip_space_and_comments (&s);
360 while (*end && *end != '>')
369 if (end > s && *end == '>')
374 result = strndup (s, end - s + 1);
376 /* Finally, collapse any whitespace that is within the message-id
382 for (r = result, len = strlen (r); *r; r++, len--)
383 if (*r == ' ' || *r == '\t')
384 memmove (r, r+1, len);
390 /* Parse a References header value, putting a copy of each referenced
391 * message-id into 'array'. */
393 parse_references (GPtrArray *array,
402 ref = parse_message_id (refs, &refs);
405 g_ptr_array_add (array, ref);
410 notmuch_database_default_path (void)
412 if (getenv ("NOTMUCH_BASE"))
413 return strdup (getenv ("NOTMUCH_BASE"));
415 return g_strdup_printf ("%s/mail", getenv ("HOME"));
419 notmuch_database_create (const char *path)
421 notmuch_database_t *notmuch = NULL;
422 char *notmuch_path = NULL;
425 char *local_path = NULL;
428 path = local_path = notmuch_database_default_path ();
430 err = stat (path, &st);
432 fprintf (stderr, "Error: Cannot create database at %s: %s.\n",
433 path, strerror (errno));
437 if (! S_ISDIR (st.st_mode)) {
438 fprintf (stderr, "Error: Cannot create database at %s: Not a directory.\n",
443 notmuch_path = g_strdup_printf ("%s/%s", path, ".notmuch");
445 err = mkdir (notmuch_path, 0755);
448 fprintf (stderr, "Error: Cannot create directory %s: %s.\n",
449 notmuch_path, strerror (errno));
453 notmuch = notmuch_database_open (path);
465 notmuch_database_open (const char *path)
467 notmuch_database_t *notmuch = NULL;
468 char *notmuch_path = NULL, *xapian_path = NULL;
471 char *local_path = NULL;
474 path = local_path = notmuch_database_default_path ();
476 notmuch_path = g_strdup_printf ("%s/%s", path, ".notmuch");
478 err = stat (notmuch_path, &st);
480 fprintf (stderr, "Error opening database at %s: %s\n",
481 notmuch_path, strerror (errno));
485 xapian_path = g_strdup_printf ("%s/%s", notmuch_path, "xapian");
487 notmuch = talloc (NULL, notmuch_database_t);
488 notmuch->path = talloc_strdup (notmuch, path);
491 notmuch->xapian_db = new Xapian::WritableDatabase (xapian_path,
492 Xapian::DB_CREATE_OR_OPEN);
493 notmuch->query_parser = new Xapian::QueryParser;
494 notmuch->query_parser->set_default_op (Xapian::Query::OP_AND);
495 notmuch->query_parser->set_database (*notmuch->xapian_db);
496 } catch (const Xapian::Error &error) {
497 fprintf (stderr, "A Xapian exception occurred: %s\n",
498 error.get_msg().c_str());
513 notmuch_database_close (notmuch_database_t *notmuch)
515 delete notmuch->query_parser;
516 delete notmuch->xapian_db;
517 talloc_free (notmuch);
521 notmuch_database_get_path (notmuch_database_t *notmuch)
523 return notmuch->path;
526 notmuch_private_status_t
527 find_timestamp_document (notmuch_database_t *notmuch, const char *db_key,
528 Xapian::Document *doc, unsigned int *doc_id)
530 return find_unique_document (notmuch, "timestamp", db_key, doc, doc_id);
533 /* We allow the user to use arbitrarily long keys for timestamps,
534 * (they're for filesystem paths after all, which have no limit we
535 * know about). But we have a term-length limit. So if we exceed that,
536 * we'll use the SHA-1 of the user's key as the actual key for
537 * constructing a database term.
539 * Caution: This function returns a newly allocated string which the
540 * caller should free() when finished.
543 timestamp_db_key (const char *key)
545 if (strlen (key) + 1 > NOTMUCH_TERM_MAX) {
546 return notmuch_sha1_of_string (key);
553 notmuch_database_set_timestamp (notmuch_database_t *notmuch,
554 const char *key, time_t timestamp)
556 Xapian::Document doc;
558 notmuch_private_status_t status;
559 notmuch_status_t ret = NOTMUCH_STATUS_SUCCESS;
562 db_key = timestamp_db_key (key);
565 status = find_timestamp_document (notmuch, db_key, &doc, &doc_id);
567 doc.add_value (0, Xapian::sortable_serialise (timestamp));
569 if (status == NOTMUCH_PRIVATE_STATUS_NO_DOCUMENT_FOUND) {
570 char *term = talloc_asprintf (NULL, "%s%s",
571 _find_prefix ("timestamp"), db_key);
575 notmuch->xapian_db->add_document (doc);
577 notmuch->xapian_db->replace_document (doc_id, doc);
580 } catch (Xapian::Error &error) {
581 fprintf (stderr, "A Xapian exception occurred: %s.\n",
582 error.get_msg().c_str());
583 ret = NOTMUCH_STATUS_XAPIAN_EXCEPTION;
593 notmuch_database_get_timestamp (notmuch_database_t *notmuch, const char *key)
595 Xapian::Document doc;
597 notmuch_private_status_t status;
601 db_key = timestamp_db_key (key);
604 status = find_timestamp_document (notmuch, db_key, &doc, &doc_id);
606 if (status == NOTMUCH_PRIVATE_STATUS_NO_DOCUMENT_FOUND)
609 ret = Xapian::sortable_unserialise (doc.get_value (0));
610 } catch (Xapian::Error &error) {
622 notmuch_database_add_message (notmuch_database_t *notmuch,
623 const char *filename)
625 notmuch_message_file_t *message_file;
626 notmuch_message_t *message;
627 notmuch_status_t ret = NOTMUCH_STATUS_SUCCESS;
629 GPtrArray *parents, *thread_ids;
631 const char *refs, *in_reply_to, *date, *header;
632 const char *from, *to, *subject, *old_filename;
637 message_file = notmuch_message_file_open (filename);
638 if (message_file == NULL) {
639 ret = NOTMUCH_STATUS_FILE_ERROR;
643 notmuch_message_file_restrict_headers (message_file,
654 /* The first order of business is to find/create a message ID. */
656 header = notmuch_message_file_get_header (message_file, "message-id");
658 message_id = parse_message_id (header, NULL);
659 /* So the header value isn't RFC-compliant, but it's
660 * better than no message-id at all. */
661 if (message_id == NULL)
662 message_id = xstrdup (header);
664 /* No message-id at all, let's generate one by taking a
665 * hash over the file's contents. */
666 char *sha1 = notmuch_sha1_of_file (filename);
668 /* If that failed too, something is really wrong. Give up. */
670 ret = NOTMUCH_STATUS_FILE_ERROR;
674 message_id = g_strdup_printf ("notmuch-sha1-%s", sha1);
678 /* Now that we have a message ID, we get a message object,
679 * (which may or may not reference an existing document in the
682 /* Use NULL for owner since we want to free this locally. */
684 /* XXX: This call can fail by either out-of-memory or an
685 * "impossible" Xapian exception. We should rewrite it to
686 * allow us to propagate the error status. */
687 message = _notmuch_message_create_for_message_id (NULL, notmuch,
689 if (message == NULL) {
690 fprintf (stderr, "Internal error. This shouldn't happen.\n\n");
691 fprintf (stderr, "I mean, it's possible you ran out of memory, but then this code path is still an internal error since it should have detected that and propagated the status value up the stack.\n");
695 /* Has a message previously been added with the same ID? */
696 old_filename = notmuch_message_get_filename (message);
697 if (old_filename && strlen (old_filename)) {
698 ret = NOTMUCH_STATUS_DUPLICATE_MESSAGE_ID;
701 _notmuch_message_set_filename (message, filename);
702 _notmuch_message_add_term (message, "type", "mail");
705 /* Next, find the thread(s) to which this message belongs. */
706 parents = g_ptr_array_new ();
708 refs = notmuch_message_file_get_header (message_file, "references");
709 parse_references (parents, refs);
711 in_reply_to = notmuch_message_file_get_header (message_file, "in-reply-to");
712 parse_references (parents, in_reply_to);
714 for (i = 0; i < parents->len; i++)
715 _notmuch_message_add_term (message, "ref",
716 (char *) g_ptr_array_index (parents, i));
718 thread_ids = find_thread_ids (notmuch, parents, message_id);
722 for (i = 0; i < parents->len; i++)
723 g_free (g_ptr_array_index (parents, i));
724 g_ptr_array_free (parents, TRUE);
726 if (thread_ids->len) {
731 for (i = 0; i < thread_ids->len; i++) {
732 id = (char *) thread_ids->pdata[i];
733 _notmuch_message_add_thread_id (message, id);
735 thread_id = g_string_new (id);
737 g_string_append_printf (thread_id, ",%s", id);
741 g_string_free (thread_id, TRUE);
743 _notmuch_message_ensure_thread_id (message);
746 g_ptr_array_free (thread_ids, TRUE);
748 date = notmuch_message_file_get_header (message_file, "date");
749 _notmuch_message_set_date (message, date);
751 from = notmuch_message_file_get_header (message_file, "from");
752 subject = notmuch_message_file_get_header (message_file, "subject");
753 to = notmuch_message_file_get_header (message_file, "to");
759 ret = NOTMUCH_STATUS_FILE_NOT_EMAIL;
762 _notmuch_message_sync (message);
764 } catch (const Xapian::Error &error) {
765 fprintf (stderr, "A Xapian exception occurred: %s.\n",
766 error.get_msg().c_str());
767 ret = NOTMUCH_STATUS_XAPIAN_EXCEPTION;
773 notmuch_message_destroy (message);
775 notmuch_message_file_close (message_file);