1 /* database.cc - The database interfaces of the notmuch mail library
3 * Copyright © 2009 Carl Worth
5 * This program is free software: you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation, either version 3 of the License, or
8 * (at your option) any later version.
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
15 * You should have received a copy of the GNU General Public License
16 * along with this program. If not, see http://www.gnu.org/licenses/ .
18 * Author: Carl Worth <cworth@cworth.org>
21 #include "database-private.h"
27 #include <glib.h> /* g_strdup_printf, g_free, GPtrArray, GHashTable */
31 #define ARRAY_SIZE(arr) (sizeof (arr) / sizeof (arr[0]))
33 /* These prefix values are specifically chosen to be compatible
34 * with sup, (http://sup.rubyforge.org), written by
35 * William Morgan <wmorgan-sup@masanjin.net>, and released
36 * under the GNU GPL v2.
44 prefix_t BOOLEAN_PREFIX_INTERNAL[] = {
48 { "timestamp", "KTS" },
51 prefix_t BOOLEAN_PREFIX_EXTERNAL[] = {
57 _find_prefix (const char *name)
61 for (i = 0; i < ARRAY_SIZE (BOOLEAN_PREFIX_INTERNAL); i++)
62 if (strcmp (name, BOOLEAN_PREFIX_INTERNAL[i].name) == 0)
63 return BOOLEAN_PREFIX_INTERNAL[i].prefix;
65 for (i = 0; i < ARRAY_SIZE (BOOLEAN_PREFIX_EXTERNAL); i++)
66 if (strcmp (name, BOOLEAN_PREFIX_EXTERNAL[i].name) == 0)
67 return BOOLEAN_PREFIX_EXTERNAL[i].prefix;
69 fprintf (stderr, "Internal error: No prefix exists for '%s'\n", name);
76 notmuch_status_to_string (notmuch_status_t status)
79 case NOTMUCH_STATUS_SUCCESS:
80 return "No error occurred";
81 case NOTMUCH_STATUS_XAPIAN_EXCEPTION:
82 return "A Xapian exception occurred";
83 case NOTMUCH_STATUS_FILE_ERROR:
84 return "Something went wrong trying to read or write a file";
85 case NOTMUCH_STATUS_FILE_NOT_EMAIL:
86 return "File is not an email";
87 case NOTMUCH_STATUS_DUPLICATE_MESSAGE_ID:
88 return "Message ID is identical to a message in database";
89 case NOTMUCH_STATUS_NULL_POINTER:
90 return "Erroneous NULL pointer";
91 case NOTMUCH_STATUS_TAG_TOO_LONG:
92 return "Tag value is too long (exceeds NOTMUCH_TAG_MAX)";
94 case NOTMUCH_STATUS_LAST_STATUS:
95 return "Unknown error status value";
99 /* XXX: We should drop this function and convert all callers to call
100 * _notmuch_message_add_term instead. */
102 add_term (Xapian::Document doc,
103 const char *prefix_name,
112 prefix = _find_prefix (prefix_name);
114 term = g_strdup_printf ("%s%s", prefix, value);
116 if (strlen (term) <= NOTMUCH_TERM_MAX)
123 find_doc_ids (notmuch_database_t *notmuch,
124 const char *prefix_name,
126 Xapian::PostingIterator *begin,
127 Xapian::PostingIterator *end)
129 Xapian::PostingIterator i;
132 term = g_strdup_printf ("%s%s", _find_prefix (prefix_name), value);
134 *begin = notmuch->xapian_db->postlist_begin (term);
136 *end = notmuch->xapian_db->postlist_end (term);
141 static notmuch_private_status_t
142 find_unique_doc_id (notmuch_database_t *notmuch,
143 const char *prefix_name,
145 unsigned int *doc_id)
147 Xapian::PostingIterator i, end;
149 find_doc_ids (notmuch, prefix_name, value, &i, &end);
153 return NOTMUCH_PRIVATE_STATUS_NO_DOCUMENT_FOUND;
156 return NOTMUCH_PRIVATE_STATUS_SUCCESS;
160 static Xapian::Document
161 find_document_for_doc_id (notmuch_database_t *notmuch, unsigned doc_id)
163 return notmuch->xapian_db->get_document (doc_id);
166 static notmuch_private_status_t
167 find_unique_document (notmuch_database_t *notmuch,
168 const char *prefix_name,
170 Xapian::Document *document,
171 unsigned int *doc_id)
173 notmuch_private_status_t status;
175 status = find_unique_doc_id (notmuch, prefix_name, value, doc_id);
178 *document = Xapian::Document ();
182 *document = find_document_for_doc_id (notmuch, *doc_id);
183 return NOTMUCH_PRIVATE_STATUS_SUCCESS;
187 insert_thread_id (GHashTable *thread_ids, Xapian::Document doc)
190 const char *value, *id, *comma;
192 value_string = doc.get_value (NOTMUCH_VALUE_THREAD);
193 value = value_string.c_str();
194 if (strlen (value)) {
197 comma = strchr (id, ',');
199 comma = id + strlen (id);
200 g_hash_table_insert (thread_ids,
201 strndup (id, comma - id), NULL);
210 notmuch_database_find_message (notmuch_database_t *notmuch,
211 const char *message_id)
213 notmuch_private_status_t status;
216 status = find_unique_doc_id (notmuch, "id", message_id, &doc_id);
218 if (status == NOTMUCH_PRIVATE_STATUS_NO_DOCUMENT_FOUND)
221 return _notmuch_message_create (notmuch, notmuch, doc_id);
224 /* Return one or more thread_ids, (as a GPtrArray of strings), for the
225 * given message based on looking into the database for any messages
226 * referenced in parents, and also for any messages in the database
227 * referencing message_id.
229 * Caller should free all strings in the array and the array itself,
230 * (g_ptr_array_free) when done. */
232 find_thread_ids (notmuch_database_t *notmuch,
234 const char *message_id)
236 Xapian::PostingIterator child, children_end;
237 Xapian::Document doc;
238 GHashTable *thread_ids;
241 const char *parent_message_id;
244 thread_ids = g_hash_table_new_full (g_str_hash, g_str_equal,
247 find_doc_ids (notmuch, "ref", message_id, &child, &children_end);
248 for ( ; child != children_end; child++) {
249 doc = find_document_for_doc_id (notmuch, *child);
250 insert_thread_id (thread_ids, doc);
253 for (i = 0; i < parents->len; i++) {
254 notmuch_message_t *parent;
255 notmuch_thread_ids_t *ids;
257 parent_message_id = (char *) g_ptr_array_index (parents, i);
258 parent = notmuch_database_find_message (notmuch, parent_message_id);
262 for (ids = notmuch_message_get_thread_ids (parent);
263 notmuch_thread_ids_has_more (ids);
264 notmuch_thread_ids_advance (ids))
268 id = notmuch_thread_ids_get (ids);
269 g_hash_table_insert (thread_ids, strdup (id), NULL);
272 notmuch_message_destroy (parent);
275 result = g_ptr_array_new ();
277 keys = g_hash_table_get_keys (thread_ids);
278 for (l = keys; l; l = l->next) {
279 char *id = (char *) l->data;
280 g_ptr_array_add (result, id);
284 /* We're done with the hash table, but we've taken the pointers to
285 * the allocated strings and put them into our result array, so
286 * tell the hash not to free them on its way out. */
287 g_hash_table_steal_all (thread_ids);
288 g_hash_table_unref (thread_ids);
293 /* Advance 'str' past any whitespace or RFC 822 comments. A comment is
294 * a (potentially nested) parenthesized sequence with '\' used to
295 * escape any character (including parentheses).
297 * If the sequence to be skipped continues to the end of the string,
298 * then 'str' will be left pointing at the final terminating '\0'
302 skip_space_and_comments (const char **str)
307 while (*s && (isspace (*s) || *s == '(')) {
308 while (*s && isspace (*s))
313 while (*s && nesting) {
329 /* Parse an RFC 822 message-id, discarding whitespace, any RFC 822
330 * comments, and the '<' and '>' delimeters.
332 * If not NULL, then *next will be made to point to the first character
333 * not parsed, (possibly pointing to the final '\0' terminator.
335 * Returns a newly allocated string which the caller should free()
338 * Returns NULL if there is any error parsing the message-id. */
340 parse_message_id (const char *message_id, const char **next)
345 if (message_id == NULL)
350 skip_space_and_comments (&s);
352 /* Skip any unstructured text as well. */
353 while (*s && *s != '<')
364 skip_space_and_comments (&s);
367 while (*end && *end != '>')
376 if (end > s && *end == '>')
381 result = strndup (s, end - s + 1);
383 /* Finally, collapse any whitespace that is within the message-id
389 for (r = result, len = strlen (r); *r; r++, len--)
390 if (*r == ' ' || *r == '\t')
391 memmove (r, r+1, len);
397 /* Parse a References header value, putting a copy of each referenced
398 * message-id into 'array'. */
400 parse_references (GPtrArray *array,
409 ref = parse_message_id (refs, &refs);
412 g_ptr_array_add (array, ref);
417 notmuch_database_default_path (void)
419 if (getenv ("NOTMUCH_BASE"))
420 return strdup (getenv ("NOTMUCH_BASE"));
422 return g_strdup_printf ("%s/mail", getenv ("HOME"));
426 notmuch_database_create (const char *path)
428 notmuch_database_t *notmuch = NULL;
429 char *notmuch_path = NULL;
432 char *local_path = NULL;
435 path = local_path = notmuch_database_default_path ();
437 err = stat (path, &st);
439 fprintf (stderr, "Error: Cannot create database at %s: %s.\n",
440 path, strerror (errno));
444 if (! S_ISDIR (st.st_mode)) {
445 fprintf (stderr, "Error: Cannot create database at %s: Not a directory.\n",
450 notmuch_path = g_strdup_printf ("%s/%s", path, ".notmuch");
452 err = mkdir (notmuch_path, 0755);
455 fprintf (stderr, "Error: Cannot create directory %s: %s.\n",
456 notmuch_path, strerror (errno));
460 notmuch = notmuch_database_open (path);
472 notmuch_database_open (const char *path)
474 notmuch_database_t *notmuch = NULL;
475 char *notmuch_path = NULL, *xapian_path = NULL;
478 char *local_path = NULL;
482 path = local_path = notmuch_database_default_path ();
484 notmuch_path = g_strdup_printf ("%s/%s", path, ".notmuch");
486 err = stat (notmuch_path, &st);
488 fprintf (stderr, "Error opening database at %s: %s\n",
489 notmuch_path, strerror (errno));
493 xapian_path = g_strdup_printf ("%s/%s", notmuch_path, "xapian");
495 notmuch = talloc (NULL, notmuch_database_t);
496 notmuch->path = talloc_strdup (notmuch, path);
499 notmuch->xapian_db = new Xapian::WritableDatabase (xapian_path,
500 Xapian::DB_CREATE_OR_OPEN);
501 notmuch->query_parser = new Xapian::QueryParser;
502 notmuch->query_parser->set_default_op (Xapian::Query::OP_AND);
503 notmuch->query_parser->set_database (*notmuch->xapian_db);
505 for (i = 0; i < ARRAY_SIZE (BOOLEAN_PREFIX_EXTERNAL); i++) {
506 prefix_t *prefix = &BOOLEAN_PREFIX_EXTERNAL[i];
507 notmuch->query_parser->add_boolean_prefix (prefix->name,
510 } catch (const Xapian::Error &error) {
511 fprintf (stderr, "A Xapian exception occurred: %s\n",
512 error.get_msg().c_str());
527 notmuch_database_close (notmuch_database_t *notmuch)
529 delete notmuch->query_parser;
530 delete notmuch->xapian_db;
531 talloc_free (notmuch);
535 notmuch_database_get_path (notmuch_database_t *notmuch)
537 return notmuch->path;
540 notmuch_private_status_t
541 find_timestamp_document (notmuch_database_t *notmuch, const char *db_key,
542 Xapian::Document *doc, unsigned int *doc_id)
544 return find_unique_document (notmuch, "timestamp", db_key, doc, doc_id);
547 /* We allow the user to use arbitrarily long keys for timestamps,
548 * (they're for filesystem paths after all, which have no limit we
549 * know about). But we have a term-length limit. So if we exceed that,
550 * we'll use the SHA-1 of the user's key as the actual key for
551 * constructing a database term.
553 * Caution: This function returns a newly allocated string which the
554 * caller should free() when finished.
557 timestamp_db_key (const char *key)
559 int term_len = strlen (_find_prefix ("timestamp")) + strlen (key);
561 if (term_len > NOTMUCH_TERM_MAX)
562 return notmuch_sha1_of_string (key);
568 notmuch_database_set_timestamp (notmuch_database_t *notmuch,
569 const char *key, time_t timestamp)
571 Xapian::Document doc;
573 notmuch_private_status_t status;
574 notmuch_status_t ret = NOTMUCH_STATUS_SUCCESS;
577 db_key = timestamp_db_key (key);
580 status = find_timestamp_document (notmuch, db_key, &doc, &doc_id);
582 doc.add_value (0, Xapian::sortable_serialise (timestamp));
584 if (status == NOTMUCH_PRIVATE_STATUS_NO_DOCUMENT_FOUND) {
585 char *term = talloc_asprintf (NULL, "%s%s",
586 _find_prefix ("timestamp"), db_key);
590 notmuch->xapian_db->add_document (doc);
592 notmuch->xapian_db->replace_document (doc_id, doc);
595 } catch (Xapian::Error &error) {
596 fprintf (stderr, "A Xapian exception occurred: %s.\n",
597 error.get_msg().c_str());
598 ret = NOTMUCH_STATUS_XAPIAN_EXCEPTION;
608 notmuch_database_get_timestamp (notmuch_database_t *notmuch, const char *key)
610 Xapian::Document doc;
612 notmuch_private_status_t status;
616 db_key = timestamp_db_key (key);
619 status = find_timestamp_document (notmuch, db_key, &doc, &doc_id);
621 if (status == NOTMUCH_PRIVATE_STATUS_NO_DOCUMENT_FOUND)
624 ret = Xapian::sortable_unserialise (doc.get_value (0));
625 } catch (Xapian::Error &error) {
637 notmuch_database_add_message (notmuch_database_t *notmuch,
638 const char *filename)
640 notmuch_message_file_t *message_file;
641 notmuch_message_t *message;
642 notmuch_status_t ret = NOTMUCH_STATUS_SUCCESS;
644 GPtrArray *parents, *thread_ids;
646 const char *refs, *in_reply_to, *date, *header;
647 const char *from, *to, *subject, *old_filename;
652 message_file = notmuch_message_file_open (filename);
653 if (message_file == NULL) {
654 ret = NOTMUCH_STATUS_FILE_ERROR;
658 notmuch_message_file_restrict_headers (message_file,
669 /* The first order of business is to find/create a message ID. */
671 header = notmuch_message_file_get_header (message_file, "message-id");
673 message_id = parse_message_id (header, NULL);
674 /* So the header value isn't RFC-compliant, but it's
675 * better than no message-id at all. */
676 if (message_id == NULL)
677 message_id = xstrdup (header);
679 /* No message-id at all, let's generate one by taking a
680 * hash over the file's contents. */
681 char *sha1 = notmuch_sha1_of_file (filename);
683 /* If that failed too, something is really wrong. Give up. */
685 ret = NOTMUCH_STATUS_FILE_ERROR;
689 message_id = g_strdup_printf ("notmuch-sha1-%s", sha1);
693 /* Now that we have a message ID, we get a message object,
694 * (which may or may not reference an existing document in the
697 /* Use NULL for owner since we want to free this locally. */
699 /* XXX: This call can fail by either out-of-memory or an
700 * "impossible" Xapian exception. We should rewrite it to
701 * allow us to propagate the error status. */
702 message = _notmuch_message_create_for_message_id (NULL, notmuch,
704 if (message == NULL) {
705 fprintf (stderr, "Internal error. This shouldn't happen.\n\n");
706 fprintf (stderr, "I mean, it's possible you ran out of memory, but then this code path is still an internal error since it should have detected that and propagated the status value up the stack.\n");
710 /* Has a message previously been added with the same ID? */
711 old_filename = notmuch_message_get_filename (message);
712 if (old_filename && strlen (old_filename)) {
713 ret = NOTMUCH_STATUS_DUPLICATE_MESSAGE_ID;
716 _notmuch_message_set_filename (message, filename);
717 _notmuch_message_add_term (message, "type", "mail");
720 /* Next, find the thread(s) to which this message belongs. */
721 parents = g_ptr_array_new ();
723 refs = notmuch_message_file_get_header (message_file, "references");
724 parse_references (parents, refs);
726 in_reply_to = notmuch_message_file_get_header (message_file, "in-reply-to");
727 parse_references (parents, in_reply_to);
729 for (i = 0; i < parents->len; i++)
730 _notmuch_message_add_term (message, "ref",
731 (char *) g_ptr_array_index (parents, i));
733 thread_ids = find_thread_ids (notmuch, parents, message_id);
737 for (i = 0; i < parents->len; i++)
738 g_free (g_ptr_array_index (parents, i));
739 g_ptr_array_free (parents, TRUE);
741 if (thread_ids->len) {
746 for (i = 0; i < thread_ids->len; i++) {
747 id = (char *) thread_ids->pdata[i];
748 _notmuch_message_add_thread_id (message, id);
750 thread_id = g_string_new (id);
752 g_string_append_printf (thread_id, ",%s", id);
756 g_string_free (thread_id, TRUE);
758 _notmuch_message_ensure_thread_id (message);
761 g_ptr_array_free (thread_ids, TRUE);
763 date = notmuch_message_file_get_header (message_file, "date");
764 _notmuch_message_set_date (message, date);
766 from = notmuch_message_file_get_header (message_file, "from");
767 subject = notmuch_message_file_get_header (message_file, "subject");
768 to = notmuch_message_file_get_header (message_file, "to");
774 ret = NOTMUCH_STATUS_FILE_NOT_EMAIL;
777 _notmuch_message_sync (message);
779 } catch (const Xapian::Error &error) {
780 fprintf (stderr, "A Xapian exception occurred: %s.\n",
781 error.get_msg().c_str());
782 ret = NOTMUCH_STATUS_XAPIAN_EXCEPTION;
788 notmuch_message_destroy (message);
790 notmuch_message_file_close (message_file);