1 #include "database-private.h"
3 /* Parse a References header value, putting a (talloc'ed under 'ctx')
4 * copy of each referenced message-id into 'hash'.
6 * We explicitly avoid including any reference identical to
7 * 'message_id' in the result (to avoid mass confusion when a single
8 * message references itself cyclically---and yes, mail messages are
9 * not infrequent in the wild that do this---don't ask me why).
11 * Return the last reference parsed, if it is not equal to message_id.
14 parse_references (void *ctx,
15 const char *message_id,
19 char *ref, *last_ref = NULL;
21 if (refs == NULL || *refs == '\0')
25 ref = _notmuch_message_id_parse (ctx, refs, &refs);
27 if (ref && strcmp (ref, message_id)) {
28 g_hash_table_add (hash, ref);
33 /* The return value of this function is used to add a parent
34 * reference to the database. We should avoid making a message
35 * its own parent, thus the above check.
37 return talloc_strdup (ctx, last_ref);
41 _notmuch_database_generate_thread_id (notmuch_database_t *notmuch)
43 /* 16 bytes (+ terminator) for hexadecimal representation of
44 * a 64-bit integer. */
45 static char thread_id[17];
47 notmuch->last_thread_id++;
49 sprintf (thread_id, "%016" PRIx64, notmuch->last_thread_id);
51 notmuch->writable_xapian_db->set_metadata ("last_thread_id", thread_id);
57 _get_metadata_thread_id_key (void *ctx, const char *message_id)
59 if (strlen (message_id) > NOTMUCH_MESSAGE_ID_MAX)
60 message_id = _notmuch_message_id_compressed (ctx, message_id);
62 return talloc_asprintf (ctx, NOTMUCH_METADATA_THREAD_ID_PREFIX "%s",
67 static notmuch_status_t
68 _resolve_message_id_to_thread_id_old (notmuch_database_t *notmuch,
70 const char *message_id,
71 const char **thread_id_ret);
74 /* Find the thread ID to which the message with 'message_id' belongs.
76 * Note: 'thread_id_ret' must not be NULL!
77 * On success '*thread_id_ret' is set to a newly talloced string belonging to
80 * Note: If there is no message in the database with the given
81 * 'message_id' then a new thread_id will be allocated for this
82 * message ID and stored in the database metadata so that the
83 * thread ID can be looked up if the message is added to the database
86 static notmuch_status_t
87 _resolve_message_id_to_thread_id (notmuch_database_t *notmuch,
89 const char *message_id,
90 const char **thread_id_ret)
92 notmuch_private_status_t status;
93 notmuch_message_t *message;
95 if (! (notmuch->features & NOTMUCH_FEATURE_GHOSTS))
96 return _resolve_message_id_to_thread_id_old (notmuch, ctx, message_id,
99 /* Look for this message (regular or ghost) */
100 message = _notmuch_message_create_for_message_id (
101 notmuch, message_id, &status);
102 if (status == NOTMUCH_PRIVATE_STATUS_SUCCESS) {
104 *thread_id_ret = talloc_steal (
105 ctx, notmuch_message_get_thread_id (message));
106 } else if (status == NOTMUCH_PRIVATE_STATUS_NO_DOCUMENT_FOUND) {
107 /* Message did not exist. Give it a fresh thread ID and
108 * populate this message as a ghost message. */
109 *thread_id_ret = talloc_strdup (
110 ctx, _notmuch_database_generate_thread_id (notmuch));
111 if (! *thread_id_ret) {
112 status = NOTMUCH_PRIVATE_STATUS_OUT_OF_MEMORY;
114 status = _notmuch_message_initialize_ghost (message, *thread_id_ret);
116 /* Commit the new ghost message */
117 _notmuch_message_sync (message);
120 /* Create failed. Fall through. */
123 notmuch_message_destroy (message);
125 return COERCE_STATUS (status, "Error creating ghost message");
128 /* Pre-ghost messages _resolve_message_id_to_thread_id */
129 static notmuch_status_t
130 _resolve_message_id_to_thread_id_old (notmuch_database_t *notmuch,
132 const char *message_id,
133 const char **thread_id_ret)
135 notmuch_status_t status;
136 notmuch_message_t *message;
137 std::string thread_id_string;
139 Xapian::WritableDatabase *db;
141 status = notmuch_database_find_message (notmuch, message_id, &message);
147 *thread_id_ret = talloc_steal (ctx,
148 notmuch_message_get_thread_id (message));
150 notmuch_message_destroy (message);
152 return NOTMUCH_STATUS_SUCCESS;
155 /* Message has not been seen yet.
157 * We may have seen a reference to it already, in which case, we
158 * can return the thread ID stored in the metadata. Otherwise, we
159 * generate a new thread ID and store it there.
161 db = notmuch->writable_xapian_db;
162 metadata_key = _get_metadata_thread_id_key (ctx, message_id);
163 thread_id_string = notmuch->xapian_db->get_metadata (metadata_key);
165 if (thread_id_string.empty ()) {
166 *thread_id_ret = talloc_strdup (ctx,
167 _notmuch_database_generate_thread_id (notmuch));
168 db->set_metadata (metadata_key, *thread_id_ret);
170 *thread_id_ret = talloc_strdup (ctx, thread_id_string.c_str ());
173 talloc_free (metadata_key);
175 return NOTMUCH_STATUS_SUCCESS;
178 static notmuch_status_t
179 _merge_threads (notmuch_database_t *notmuch,
180 const char *winner_thread_id,
181 const char *loser_thread_id)
183 Xapian::PostingIterator loser, loser_end;
184 notmuch_message_t *message = NULL;
185 notmuch_private_status_t private_status;
186 notmuch_status_t ret = NOTMUCH_STATUS_SUCCESS;
188 _notmuch_database_find_doc_ids (notmuch, "thread", loser_thread_id, &loser, &loser_end);
190 for (; loser != loser_end; loser++) {
191 message = _notmuch_message_create (notmuch, notmuch,
192 *loser, &private_status);
193 if (message == NULL) {
194 ret = COERCE_STATUS (private_status,
195 "Cannot find document for doc_id from query");
199 _notmuch_message_remove_term (message, "thread", loser_thread_id);
200 _notmuch_message_add_term (message, "thread", winner_thread_id);
201 _notmuch_message_sync (message);
203 notmuch_message_destroy (message);
209 notmuch_message_destroy (message);
215 _my_talloc_free_for_g_hash (void *ptr)
221 _notmuch_database_link_message_to_parents (notmuch_database_t *notmuch,
222 notmuch_message_t *message,
223 notmuch_message_file_t *message_file,
224 const char **thread_id)
226 GHashTable *parents = NULL;
227 const char *refs, *in_reply_to, *in_reply_to_message_id, *strict_message_id = NULL;
228 const char *last_ref_message_id, *this_message_id;
229 GList *l, *keys = NULL;
230 notmuch_status_t ret = NOTMUCH_STATUS_SUCCESS;
232 parents = g_hash_table_new_full (g_str_hash, g_str_equal,
233 _my_talloc_free_for_g_hash, NULL);
234 this_message_id = notmuch_message_get_message_id (message);
236 refs = _notmuch_message_file_get_header (message_file, "references");
237 last_ref_message_id = parse_references (message,
241 in_reply_to = _notmuch_message_file_get_header (message_file, "in-reply-to");
243 strict_message_id = _notmuch_message_id_parse_strict (message,
246 in_reply_to_message_id = parse_references (message,
248 parents, in_reply_to);
250 /* For the parent of this message, use
251 * 1) the In-Reply-To header, if it looks sane, otherwise
252 * 2) the last message ID of the References header, if available.
253 * 3) Otherwise, fall back to the first message ID in
254 * the In-Reply-To header.
257 if (strict_message_id) {
258 _notmuch_message_add_term (message, "replyto", strict_message_id);
259 } else if (last_ref_message_id) {
260 _notmuch_message_add_term (message, "replyto",
261 last_ref_message_id);
262 } else if (in_reply_to_message_id) {
263 _notmuch_message_add_term (message, "replyto",
264 in_reply_to_message_id);
267 keys = g_hash_table_get_keys (parents);
268 for (l = keys; l; l = l->next) {
269 char *parent_message_id;
270 const char *parent_thread_id = NULL;
272 parent_message_id = (char *) l->data;
274 _notmuch_message_add_term (message, "reference",
277 ret = _resolve_message_id_to_thread_id (notmuch,
284 if (*thread_id == NULL) {
285 *thread_id = talloc_strdup (message, parent_thread_id);
286 _notmuch_message_add_term (message, "thread", *thread_id);
287 } else if (strcmp (*thread_id, parent_thread_id)) {
288 ret = _merge_threads (notmuch, *thread_id, parent_thread_id);
298 g_hash_table_unref (parents);
303 static notmuch_status_t
304 _notmuch_database_link_message_to_children (notmuch_database_t *notmuch,
305 notmuch_message_t *message,
306 const char **thread_id)
308 const char *message_id = notmuch_message_get_message_id (message);
309 Xapian::PostingIterator child, children_end;
310 notmuch_message_t *child_message = NULL;
311 const char *child_thread_id;
312 notmuch_status_t ret = NOTMUCH_STATUS_SUCCESS;
313 notmuch_private_status_t private_status;
315 _notmuch_database_find_doc_ids (notmuch, "reference", message_id, &child, &children_end);
317 for (; child != children_end; child++) {
319 child_message = _notmuch_message_create (message, notmuch,
320 *child, &private_status);
321 if (child_message == NULL) {
322 ret = COERCE_STATUS (private_status,
323 "Cannot find document for doc_id from query");
327 child_thread_id = notmuch_message_get_thread_id (child_message);
328 if (*thread_id == NULL) {
329 *thread_id = talloc_strdup (message, child_thread_id);
330 _notmuch_message_add_term (message, "thread", *thread_id);
331 } else if (strcmp (*thread_id, child_thread_id)) {
332 _notmuch_message_remove_term (child_message, "reference",
334 _notmuch_message_sync (child_message);
335 ret = _merge_threads (notmuch, *thread_id, child_thread_id);
340 notmuch_message_destroy (child_message);
341 child_message = NULL;
346 notmuch_message_destroy (child_message);
351 /* Fetch and clear the stored thread_id for message, or NULL if none. */
353 _consume_metadata_thread_id (void *ctx, notmuch_database_t *notmuch,
354 notmuch_message_t *message)
356 const char *message_id;
357 std::string stored_id;
360 message_id = notmuch_message_get_message_id (message);
361 metadata_key = _get_metadata_thread_id_key (ctx, message_id);
363 /* Check if we have already seen related messages to this one.
364 * If we have then use the thread_id that we stored at that time.
366 stored_id = notmuch->xapian_db->get_metadata (metadata_key);
367 if (stored_id.empty ()) {
370 /* Clear the metadata for this message ID. We don't need it
372 notmuch->writable_xapian_db->set_metadata (metadata_key, "");
374 return talloc_strdup (ctx, stored_id.c_str ());
378 /* Given a blank or ghost 'message' and its corresponding
379 * 'message_file' link it to existing threads in the database.
381 * First, if is_ghost, this retrieves the thread ID already stored in
382 * the message (which will be the case if a message was previously
383 * added that referenced this one). If the message is blank
384 * (!is_ghost), it doesn't have a thread ID yet (we'll generate one
385 * later in this function). If the database does not support ghost
386 * messages, this checks for a thread ID stored in database metadata
387 * for this message ID.
389 * Second, we look at 'message_file' and its link-relevant headers
390 * (References and In-Reply-To) for message IDs.
392 * Finally, we look in the database for existing message that
393 * reference 'message'.
395 * In all cases, we assign to the current message the first thread ID
396 * found. We will also merge any existing, distinct threads where this
397 * message belongs to both, (which is not uncommon when messages are
398 * processed out of order).
400 * Finally, if no thread ID has been found through referenced messages, we
401 * call _notmuch_message_generate_thread_id to generate a new thread
402 * ID. This should only happen for new, top-level messages, (no
403 * References or In-Reply-To header in this message, and no previously
404 * added message refers to this message).
406 static notmuch_status_t
407 _notmuch_database_link_message (notmuch_database_t *notmuch,
408 notmuch_message_t *message,
409 notmuch_message_file_t *message_file,
413 void *local = talloc_new (NULL);
414 notmuch_status_t status;
415 const char *thread_id = NULL;
417 /* Check if the message already had a thread ID */
419 thread_id = notmuch_message_get_thread_id (message);
420 } else if (notmuch->features & NOTMUCH_FEATURE_GHOSTS) {
422 thread_id = notmuch_message_get_thread_id (message);
424 thread_id = _consume_metadata_thread_id (local, notmuch, message);
426 _notmuch_message_add_term (message, "thread", thread_id);
429 status = _notmuch_database_link_message_to_parents (notmuch, message,
435 if (! (notmuch->features & NOTMUCH_FEATURE_GHOSTS)) {
436 /* In general, it shouldn't be necessary to link children,
437 * since the earlier indexing of those children will have
438 * stored a thread ID for the missing parent. However, prior
439 * to ghost messages, these stored thread IDs were NOT
440 * rewritten during thread merging (and there was no
441 * performant way to do so), so if indexed children were
442 * pulled into a different thread ID by a merge, it was
443 * necessary to pull them *back* into the stored thread ID of
444 * the parent. With ghost messages, we just rewrite the
445 * stored thread IDs during merging, so this workaround isn't
447 status = _notmuch_database_link_message_to_children (notmuch, message,
453 /* If not part of any existing thread, generate a new thread ID. */
454 if (thread_id == NULL) {
455 thread_id = _notmuch_database_generate_thread_id (notmuch);
457 _notmuch_message_add_term (message, "thread", thread_id);
467 notmuch_database_index_file (notmuch_database_t *notmuch,
468 const char *filename,
469 notmuch_indexopts_t *indexopts,
470 notmuch_message_t **message_ret)
472 notmuch_message_file_t *message_file;
473 notmuch_message_t *message = NULL;
474 notmuch_status_t ret = NOTMUCH_STATUS_SUCCESS, ret2;
475 notmuch_private_status_t private_status;
476 notmuch_bool_t is_ghost = false, is_new = false;
477 notmuch_indexopts_t *def_indexopts = NULL;
480 const char *from, *to, *subject;
481 char *message_id = NULL;
486 ret = _notmuch_database_ensure_writable (notmuch);
490 message_file = _notmuch_message_file_open (notmuch, filename);
491 if (message_file == NULL)
492 return NOTMUCH_STATUS_FILE_ERROR;
494 /* Adding a message may change many documents. Do this all
496 ret = notmuch_database_begin_atomic (notmuch);
500 ret = _notmuch_message_file_get_headers (message_file,
501 &from, &subject, &to, &date,
507 /* Now that we have a message ID, we get a message object,
508 * (which may or may not reference an existing document in the
511 message = _notmuch_message_create_for_message_id (notmuch,
515 talloc_free (message_id);
517 /* We cannot call notmuch_message_get_flag for a new message */
518 switch (private_status) {
519 case NOTMUCH_PRIVATE_STATUS_NO_DOCUMENT_FOUND:
523 case NOTMUCH_PRIVATE_STATUS_SUCCESS:
524 ret = notmuch_message_get_flag_st (message, NOTMUCH_MESSAGE_FLAG_GHOST, &is_ghost);
530 ret = COERCE_STATUS (private_status,
531 "Unexpected status value from _notmuch_message_create_for_message_id");
535 _notmuch_message_add_filename (message, filename);
537 if (is_new || is_ghost) {
538 _notmuch_message_add_term (message, "type", "mail");
540 /* Convert ghost message to a regular message */
541 _notmuch_message_remove_term (message, "type", "ghost");
544 ret = _notmuch_database_link_message (notmuch, message,
545 message_file, is_ghost, is_new);
549 if (is_new || is_ghost)
550 _notmuch_message_set_header_values (message, date, from, subject);
553 def_indexopts = notmuch_database_get_default_indexopts (notmuch);
554 indexopts = def_indexopts;
557 ret = _notmuch_message_index_file (message, indexopts, message_file);
561 if (! is_new && ! is_ghost)
562 ret = NOTMUCH_STATUS_DUPLICATE_MESSAGE_ID;
564 _notmuch_message_sync (message);
565 } catch (const Xapian::Error &error) {
566 _notmuch_database_log (notmuch, "A Xapian exception occurred adding message: %s.\n",
567 error.get_msg ().c_str ());
568 notmuch->exception_reported = true;
569 ret = NOTMUCH_STATUS_XAPIAN_EXCEPTION;
575 notmuch_indexopts_destroy (def_indexopts);
578 if ((ret == NOTMUCH_STATUS_SUCCESS ||
579 ret == NOTMUCH_STATUS_DUPLICATE_MESSAGE_ID) && message_ret)
580 *message_ret = message;
582 notmuch_message_destroy (message);
586 _notmuch_message_file_close (message_file);
588 ret2 = notmuch_database_end_atomic (notmuch);
589 if ((ret == NOTMUCH_STATUS_SUCCESS ||
590 ret == NOTMUCH_STATUS_DUPLICATE_MESSAGE_ID) &&
591 ret2 != NOTMUCH_STATUS_SUCCESS)
598 notmuch_database_add_message (notmuch_database_t *notmuch,
599 const char *filename,
600 notmuch_message_t **message_ret)
602 return notmuch_database_index_file (notmuch, filename,