1 /* database.cc - The database interfaces of the notmuch mail library
3 * Copyright © 2009 Carl Worth
5 * This program is free software: you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation, either version 3 of the License, or
8 * (at your option) any later version.
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
15 * You should have received a copy of the GNU General Public License
16 * along with this program. If not, see http://www.gnu.org/licenses/ .
18 * Author: Carl Worth <cworth@cworth.org>
21 #include "database-private.h"
27 #include <glib.h> /* g_strdup_printf, g_free, GPtrArray, GHashTable */
31 #define ARRAY_SIZE(arr) (sizeof (arr) / sizeof (arr[0]))
33 /* These prefix values are specifically chosen to be compatible
34 * with sup, (http://sup.rubyforge.org), written by
35 * William Morgan <wmorgan-sup@masanjin.net>, and released
36 * under the GNU GPL v2.
44 prefix_t BOOLEAN_PREFIX[] = {
51 { "timestamp", "KTS" },
55 _find_prefix (const char *name)
59 for (i = 0; i < ARRAY_SIZE (BOOLEAN_PREFIX); i++)
60 if (strcmp (name, BOOLEAN_PREFIX[i].name) == 0)
61 return BOOLEAN_PREFIX[i].prefix;
63 fprintf (stderr, "Internal error: No prefix exists for '%s'\n", name);
70 notmuch_status_to_string (notmuch_status_t status)
73 case NOTMUCH_STATUS_SUCCESS:
74 return "No error occurred";
75 case NOTMUCH_STATUS_XAPIAN_EXCEPTION:
76 return "A Xapian exception occurred";
77 case NOTMUCH_STATUS_FILE_ERROR:
78 return "Something went wrong trying to read or write a file";
79 case NOTMUCH_STATUS_FILE_NOT_EMAIL:
80 return "File is not an email";
81 case NOTMUCH_STATUS_DUPLICATE_MESSAGE_ID:
82 return "Message ID is identical to a message in database";
83 case NOTMUCH_STATUS_NULL_POINTER:
84 return "Erroneous NULL pointer";
85 case NOTMUCH_STATUS_TAG_TOO_LONG:
86 return "Tag value is too long (exceeds NOTMUCH_TAG_MAX)";
88 case NOTMUCH_STATUS_LAST_STATUS:
89 return "Unknown error status value";
93 /* XXX: We should drop this function and convert all callers to call
94 * _notmuch_message_add_term instead. */
96 add_term (Xapian::Document doc,
97 const char *prefix_name,
106 prefix = _find_prefix (prefix_name);
108 term = g_strdup_printf ("%s%s", prefix, value);
110 if (strlen (term) <= NOTMUCH_TERM_MAX)
117 find_doc_ids (notmuch_database_t *notmuch,
118 const char *prefix_name,
120 Xapian::PostingIterator *begin,
121 Xapian::PostingIterator *end)
123 Xapian::PostingIterator i;
126 term = g_strdup_printf ("%s%s", _find_prefix (prefix_name), value);
128 *begin = notmuch->xapian_db->postlist_begin (term);
130 *end = notmuch->xapian_db->postlist_end (term);
135 static notmuch_private_status_t
136 find_unique_doc_id (notmuch_database_t *notmuch,
137 const char *prefix_name,
139 unsigned int *doc_id)
141 Xapian::PostingIterator i, end;
143 find_doc_ids (notmuch, prefix_name, value, &i, &end);
147 return NOTMUCH_PRIVATE_STATUS_NO_DOCUMENT_FOUND;
150 return NOTMUCH_PRIVATE_STATUS_SUCCESS;
154 static Xapian::Document
155 find_document_for_doc_id (notmuch_database_t *notmuch, unsigned doc_id)
157 return notmuch->xapian_db->get_document (doc_id);
160 static notmuch_private_status_t
161 find_unique_document (notmuch_database_t *notmuch,
162 const char *prefix_name,
164 Xapian::Document *document,
165 unsigned int *doc_id)
167 notmuch_private_status_t status;
169 status = find_unique_doc_id (notmuch, prefix_name, value, doc_id);
172 *document = Xapian::Document ();
176 *document = find_document_for_doc_id (notmuch, *doc_id);
177 return NOTMUCH_PRIVATE_STATUS_SUCCESS;
181 insert_thread_id (GHashTable *thread_ids, Xapian::Document doc)
184 const char *value, *id, *comma;
186 value_string = doc.get_value (NOTMUCH_VALUE_THREAD);
187 value = value_string.c_str();
188 if (strlen (value)) {
191 comma = strchr (id, ',');
193 comma = id + strlen (id);
194 g_hash_table_insert (thread_ids,
195 strndup (id, comma - id), NULL);
204 notmuch_database_find_message (notmuch_database_t *notmuch,
205 const char *message_id)
207 notmuch_private_status_t status;
210 status = find_unique_doc_id (notmuch, "msgid", message_id, &doc_id);
212 if (status == NOTMUCH_PRIVATE_STATUS_NO_DOCUMENT_FOUND)
215 return _notmuch_message_create (notmuch, notmuch, doc_id);
218 /* Return one or more thread_ids, (as a GPtrArray of strings), for the
219 * given message based on looking into the database for any messages
220 * referenced in parents, and also for any messages in the database
221 * referencing message_id.
223 * Caller should free all strings in the array and the array itself,
224 * (g_ptr_array_free) when done. */
226 find_thread_ids (notmuch_database_t *notmuch,
228 const char *message_id)
230 Xapian::PostingIterator child, children_end;
231 Xapian::Document doc;
232 GHashTable *thread_ids;
235 const char *parent_message_id;
238 thread_ids = g_hash_table_new_full (g_str_hash, g_str_equal,
241 find_doc_ids (notmuch, "ref", message_id, &child, &children_end);
242 for ( ; child != children_end; child++) {
243 doc = find_document_for_doc_id (notmuch, *child);
244 insert_thread_id (thread_ids, doc);
247 for (i = 0; i < parents->len; i++) {
248 notmuch_message_t *parent;
249 notmuch_thread_ids_t *ids;
251 parent_message_id = (char *) g_ptr_array_index (parents, i);
252 parent = notmuch_database_find_message (notmuch, parent_message_id);
256 for (ids = notmuch_message_get_thread_ids (parent);
257 notmuch_thread_ids_has_more (ids);
258 notmuch_thread_ids_advance (ids))
262 id = notmuch_thread_ids_get (ids);
263 g_hash_table_insert (thread_ids, strdup (id), NULL);
266 notmuch_message_destroy (parent);
269 result = g_ptr_array_new ();
271 keys = g_hash_table_get_keys (thread_ids);
272 for (l = keys; l; l = l->next) {
273 char *id = (char *) l->data;
274 g_ptr_array_add (result, id);
278 /* We're done with the hash table, but we've taken the pointers to
279 * the allocated strings and put them into our result array, so
280 * tell the hash not to free them on its way out. */
281 g_hash_table_steal_all (thread_ids);
282 g_hash_table_unref (thread_ids);
287 /* Advance 'str' past any whitespace or RFC 822 comments. A comment is
288 * a (potentially nested) parenthesized sequence with '\' used to
289 * escape any character (including parentheses).
291 * If the sequence to be skipped continues to the end of the string,
292 * then 'str' will be left pointing at the final terminating '\0'
296 skip_space_and_comments (const char **str)
301 while (*s && (isspace (*s) || *s == '(')) {
302 while (*s && isspace (*s))
307 while (*s && nesting) {
323 /* Parse an RFC 822 message-id, discarding whitespace, any RFC 822
324 * comments, and the '<' and '>' delimeters.
326 * If not NULL, then *next will be made to point to the first character
327 * not parsed, (possibly pointing to the final '\0' terminator.
329 * Returns a newly allocated string which the caller should free()
332 * Returns NULL if there is any error parsing the message-id. */
334 parse_message_id (const char *message_id, const char **next)
339 if (message_id == NULL)
344 skip_space_and_comments (&s);
346 /* Skip any unstructured text as well. */
347 while (*s && *s != '<')
358 skip_space_and_comments (&s);
361 while (*end && *end != '>')
370 if (end > s && *end == '>')
375 result = strndup (s, end - s + 1);
377 /* Finally, collapse any whitespace that is within the message-id
383 for (r = result, len = strlen (r); *r; r++, len--)
384 if (*r == ' ' || *r == '\t')
385 memmove (r, r+1, len);
391 /* Parse a References header value, putting a copy of each referenced
392 * message-id into 'array'. */
394 parse_references (GPtrArray *array,
403 ref = parse_message_id (refs, &refs);
406 g_ptr_array_add (array, ref);
411 notmuch_database_default_path (void)
413 if (getenv ("NOTMUCH_BASE"))
414 return strdup (getenv ("NOTMUCH_BASE"));
416 return g_strdup_printf ("%s/mail", getenv ("HOME"));
420 notmuch_database_create (const char *path)
422 notmuch_database_t *notmuch = NULL;
423 char *notmuch_path = NULL;
426 char *local_path = NULL;
429 path = local_path = notmuch_database_default_path ();
431 err = stat (path, &st);
433 fprintf (stderr, "Error: Cannot create database at %s: %s.\n",
434 path, strerror (errno));
438 if (! S_ISDIR (st.st_mode)) {
439 fprintf (stderr, "Error: Cannot create database at %s: Not a directory.\n",
444 notmuch_path = g_strdup_printf ("%s/%s", path, ".notmuch");
446 err = mkdir (notmuch_path, 0755);
449 fprintf (stderr, "Error: Cannot create directory %s: %s.\n",
450 notmuch_path, strerror (errno));
454 notmuch = notmuch_database_open (path);
466 notmuch_database_open (const char *path)
468 notmuch_database_t *notmuch = NULL;
469 char *notmuch_path = NULL, *xapian_path = NULL;
472 char *local_path = NULL;
475 path = local_path = notmuch_database_default_path ();
477 notmuch_path = g_strdup_printf ("%s/%s", path, ".notmuch");
479 err = stat (notmuch_path, &st);
481 fprintf (stderr, "Error opening database at %s: %s\n",
482 notmuch_path, strerror (errno));
486 xapian_path = g_strdup_printf ("%s/%s", notmuch_path, "xapian");
488 notmuch = talloc (NULL, notmuch_database_t);
489 notmuch->path = talloc_strdup (notmuch, path);
492 notmuch->xapian_db = new Xapian::WritableDatabase (xapian_path,
493 Xapian::DB_CREATE_OR_OPEN);
494 notmuch->query_parser = new Xapian::QueryParser;
495 notmuch->query_parser->set_default_op (Xapian::Query::OP_AND);
496 notmuch->query_parser->set_database (*notmuch->xapian_db);
497 notmuch->query_parser->add_boolean_prefix ("id", _find_prefix ("id"));
498 notmuch->query_parser->add_boolean_prefix ("tag", _find_prefix ("tag"));
499 notmuch->query_parser->add_boolean_prefix ("type", _find_prefix ("type"));
500 } catch (const Xapian::Error &error) {
501 fprintf (stderr, "A Xapian exception occurred: %s\n",
502 error.get_msg().c_str());
517 notmuch_database_close (notmuch_database_t *notmuch)
519 delete notmuch->query_parser;
520 delete notmuch->xapian_db;
521 talloc_free (notmuch);
525 notmuch_database_get_path (notmuch_database_t *notmuch)
527 return notmuch->path;
530 notmuch_private_status_t
531 find_timestamp_document (notmuch_database_t *notmuch, const char *db_key,
532 Xapian::Document *doc, unsigned int *doc_id)
534 return find_unique_document (notmuch, "timestamp", db_key, doc, doc_id);
537 /* We allow the user to use arbitrarily long keys for timestamps,
538 * (they're for filesystem paths after all, which have no limit we
539 * know about). But we have a term-length limit. So if we exceed that,
540 * we'll use the SHA-1 of the user's key as the actual key for
541 * constructing a database term.
543 * Caution: This function returns a newly allocated string which the
544 * caller should free() when finished.
547 timestamp_db_key (const char *key)
549 int term_len = strlen (_find_prefix ("timestamp")) + strlen (key);
551 if (term_len > NOTMUCH_TERM_MAX)
552 return notmuch_sha1_of_string (key);
558 notmuch_database_set_timestamp (notmuch_database_t *notmuch,
559 const char *key, time_t timestamp)
561 Xapian::Document doc;
563 notmuch_private_status_t status;
564 notmuch_status_t ret = NOTMUCH_STATUS_SUCCESS;
567 db_key = timestamp_db_key (key);
570 status = find_timestamp_document (notmuch, db_key, &doc, &doc_id);
572 doc.add_value (0, Xapian::sortable_serialise (timestamp));
574 if (status == NOTMUCH_PRIVATE_STATUS_NO_DOCUMENT_FOUND) {
575 char *term = talloc_asprintf (NULL, "%s%s",
576 _find_prefix ("timestamp"), db_key);
580 notmuch->xapian_db->add_document (doc);
582 notmuch->xapian_db->replace_document (doc_id, doc);
585 } catch (Xapian::Error &error) {
586 fprintf (stderr, "A Xapian exception occurred: %s.\n",
587 error.get_msg().c_str());
588 ret = NOTMUCH_STATUS_XAPIAN_EXCEPTION;
598 notmuch_database_get_timestamp (notmuch_database_t *notmuch, const char *key)
600 Xapian::Document doc;
602 notmuch_private_status_t status;
606 db_key = timestamp_db_key (key);
609 status = find_timestamp_document (notmuch, db_key, &doc, &doc_id);
611 if (status == NOTMUCH_PRIVATE_STATUS_NO_DOCUMENT_FOUND)
614 ret = Xapian::sortable_unserialise (doc.get_value (0));
615 } catch (Xapian::Error &error) {
627 notmuch_database_add_message (notmuch_database_t *notmuch,
628 const char *filename)
630 notmuch_message_file_t *message_file;
631 notmuch_message_t *message;
632 notmuch_status_t ret = NOTMUCH_STATUS_SUCCESS;
634 GPtrArray *parents, *thread_ids;
636 const char *refs, *in_reply_to, *date, *header;
637 const char *from, *to, *subject, *old_filename;
642 message_file = notmuch_message_file_open (filename);
643 if (message_file == NULL) {
644 ret = NOTMUCH_STATUS_FILE_ERROR;
648 notmuch_message_file_restrict_headers (message_file,
659 /* The first order of business is to find/create a message ID. */
661 header = notmuch_message_file_get_header (message_file, "message-id");
663 message_id = parse_message_id (header, NULL);
664 /* So the header value isn't RFC-compliant, but it's
665 * better than no message-id at all. */
666 if (message_id == NULL)
667 message_id = xstrdup (header);
669 /* No message-id at all, let's generate one by taking a
670 * hash over the file's contents. */
671 char *sha1 = notmuch_sha1_of_file (filename);
673 /* If that failed too, something is really wrong. Give up. */
675 ret = NOTMUCH_STATUS_FILE_ERROR;
679 message_id = g_strdup_printf ("notmuch-sha1-%s", sha1);
683 /* Now that we have a message ID, we get a message object,
684 * (which may or may not reference an existing document in the
687 /* Use NULL for owner since we want to free this locally. */
689 /* XXX: This call can fail by either out-of-memory or an
690 * "impossible" Xapian exception. We should rewrite it to
691 * allow us to propagate the error status. */
692 message = _notmuch_message_create_for_message_id (NULL, notmuch,
694 if (message == NULL) {
695 fprintf (stderr, "Internal error. This shouldn't happen.\n\n");
696 fprintf (stderr, "I mean, it's possible you ran out of memory, but then this code path is still an internal error since it should have detected that and propagated the status value up the stack.\n");
700 /* Has a message previously been added with the same ID? */
701 old_filename = notmuch_message_get_filename (message);
702 if (old_filename && strlen (old_filename)) {
703 ret = NOTMUCH_STATUS_DUPLICATE_MESSAGE_ID;
706 _notmuch_message_set_filename (message, filename);
707 _notmuch_message_add_term (message, "type", "mail");
710 /* Next, find the thread(s) to which this message belongs. */
711 parents = g_ptr_array_new ();
713 refs = notmuch_message_file_get_header (message_file, "references");
714 parse_references (parents, refs);
716 in_reply_to = notmuch_message_file_get_header (message_file, "in-reply-to");
717 parse_references (parents, in_reply_to);
719 for (i = 0; i < parents->len; i++)
720 _notmuch_message_add_term (message, "ref",
721 (char *) g_ptr_array_index (parents, i));
723 thread_ids = find_thread_ids (notmuch, parents, message_id);
727 for (i = 0; i < parents->len; i++)
728 g_free (g_ptr_array_index (parents, i));
729 g_ptr_array_free (parents, TRUE);
731 if (thread_ids->len) {
736 for (i = 0; i < thread_ids->len; i++) {
737 id = (char *) thread_ids->pdata[i];
738 _notmuch_message_add_thread_id (message, id);
740 thread_id = g_string_new (id);
742 g_string_append_printf (thread_id, ",%s", id);
746 g_string_free (thread_id, TRUE);
748 _notmuch_message_ensure_thread_id (message);
751 g_ptr_array_free (thread_ids, TRUE);
753 date = notmuch_message_file_get_header (message_file, "date");
754 _notmuch_message_set_date (message, date);
756 from = notmuch_message_file_get_header (message_file, "from");
757 subject = notmuch_message_file_get_header (message_file, "subject");
758 to = notmuch_message_file_get_header (message_file, "to");
764 ret = NOTMUCH_STATUS_FILE_NOT_EMAIL;
767 _notmuch_message_sync (message);
769 } catch (const Xapian::Error &error) {
770 fprintf (stderr, "A Xapian exception occurred: %s.\n",
771 error.get_msg().c_str());
772 ret = NOTMUCH_STATUS_XAPIAN_EXCEPTION;
778 notmuch_message_destroy (message);
780 notmuch_message_file_close (message_file);