1 #include "database-private.h"
3 /* Parse a References header value, putting a (talloc'ed under 'ctx')
4 * copy of each referenced message-id into 'hash'.
6 * We explicitly avoid including any reference identical to
7 * 'message_id' in the result (to avoid mass confusion when a single
8 * message references itself cyclically---and yes, mail messages are
9 * not infrequent in the wild that do this---don't ask me why).
11 * Return the last reference parsed, if it is not equal to message_id.
14 parse_references (void *ctx,
15 const char *message_id,
19 char *ref, *last_ref = NULL;
21 if (refs == NULL || *refs == '\0')
25 ref = _notmuch_message_id_parse (ctx, refs, &refs);
27 if (ref && strcmp (ref, message_id)) {
28 g_hash_table_add (hash, ref);
33 /* The return value of this function is used to add a parent
34 * reference to the database. We should avoid making a message
35 * its own parent, thus the above check.
37 return talloc_strdup(ctx, last_ref);
41 _notmuch_database_generate_thread_id (notmuch_database_t *notmuch)
43 /* 16 bytes (+ terminator) for hexadecimal representation of
44 * a 64-bit integer. */
45 static char thread_id[17];
46 Xapian::WritableDatabase *db;
48 db = static_cast <Xapian::WritableDatabase *> (notmuch->xapian_db);
50 notmuch->last_thread_id++;
52 sprintf (thread_id, "%016" PRIx64, notmuch->last_thread_id);
54 db->set_metadata ("last_thread_id", thread_id);
60 _get_metadata_thread_id_key (void *ctx, const char *message_id)
62 if (strlen (message_id) > NOTMUCH_MESSAGE_ID_MAX)
63 message_id = _notmuch_message_id_compressed (ctx, message_id);
65 return talloc_asprintf (ctx, NOTMUCH_METADATA_THREAD_ID_PREFIX "%s",
70 static notmuch_status_t
71 _resolve_message_id_to_thread_id_old (notmuch_database_t *notmuch,
73 const char *message_id,
74 const char **thread_id_ret);
77 /* Find the thread ID to which the message with 'message_id' belongs.
79 * Note: 'thread_id_ret' must not be NULL!
80 * On success '*thread_id_ret' is set to a newly talloced string belonging to
83 * Note: If there is no message in the database with the given
84 * 'message_id' then a new thread_id will be allocated for this
85 * message ID and stored in the database metadata so that the
86 * thread ID can be looked up if the message is added to the database
89 static notmuch_status_t
90 _resolve_message_id_to_thread_id (notmuch_database_t *notmuch,
92 const char *message_id,
93 const char **thread_id_ret)
95 notmuch_private_status_t status;
96 notmuch_message_t *message;
98 if (! (notmuch->features & NOTMUCH_FEATURE_GHOSTS))
99 return _resolve_message_id_to_thread_id_old (notmuch, ctx, message_id,
102 /* Look for this message (regular or ghost) */
103 message = _notmuch_message_create_for_message_id (
104 notmuch, message_id, &status);
105 if (status == NOTMUCH_PRIVATE_STATUS_SUCCESS) {
107 *thread_id_ret = talloc_steal (
108 ctx, notmuch_message_get_thread_id (message));
109 } else if (status == NOTMUCH_PRIVATE_STATUS_NO_DOCUMENT_FOUND) {
110 /* Message did not exist. Give it a fresh thread ID and
111 * populate this message as a ghost message. */
112 *thread_id_ret = talloc_strdup (
113 ctx, _notmuch_database_generate_thread_id (notmuch));
114 if (! *thread_id_ret) {
115 status = NOTMUCH_PRIVATE_STATUS_OUT_OF_MEMORY;
117 status = _notmuch_message_initialize_ghost (message, *thread_id_ret);
119 /* Commit the new ghost message */
120 _notmuch_message_sync (message);
123 /* Create failed. Fall through. */
126 notmuch_message_destroy (message);
128 return COERCE_STATUS (status, "Error creating ghost message");
131 /* Pre-ghost messages _resolve_message_id_to_thread_id */
132 static notmuch_status_t
133 _resolve_message_id_to_thread_id_old (notmuch_database_t *notmuch,
135 const char *message_id,
136 const char **thread_id_ret)
138 notmuch_status_t status;
139 notmuch_message_t *message;
140 std::string thread_id_string;
142 Xapian::WritableDatabase *db;
144 status = notmuch_database_find_message (notmuch, message_id, &message);
150 *thread_id_ret = talloc_steal (ctx,
151 notmuch_message_get_thread_id (message));
153 notmuch_message_destroy (message);
155 return NOTMUCH_STATUS_SUCCESS;
158 /* Message has not been seen yet.
160 * We may have seen a reference to it already, in which case, we
161 * can return the thread ID stored in the metadata. Otherwise, we
162 * generate a new thread ID and store it there.
164 db = static_cast <Xapian::WritableDatabase *> (notmuch->xapian_db);
165 metadata_key = _get_metadata_thread_id_key (ctx, message_id);
166 thread_id_string = notmuch->xapian_db->get_metadata (metadata_key);
168 if (thread_id_string.empty()) {
169 *thread_id_ret = talloc_strdup (ctx,
170 _notmuch_database_generate_thread_id (notmuch));
171 db->set_metadata (metadata_key, *thread_id_ret);
173 *thread_id_ret = talloc_strdup (ctx, thread_id_string.c_str());
176 talloc_free (metadata_key);
178 return NOTMUCH_STATUS_SUCCESS;
181 static notmuch_status_t
182 _merge_threads (notmuch_database_t *notmuch,
183 const char *winner_thread_id,
184 const char *loser_thread_id)
186 Xapian::PostingIterator loser, loser_end;
187 notmuch_message_t *message = NULL;
188 notmuch_private_status_t private_status;
189 notmuch_status_t ret = NOTMUCH_STATUS_SUCCESS;
191 _notmuch_database_find_doc_ids (notmuch, "thread", loser_thread_id, &loser, &loser_end);
193 for ( ; loser != loser_end; loser++) {
194 message = _notmuch_message_create (notmuch, notmuch,
195 *loser, &private_status);
196 if (message == NULL) {
197 ret = COERCE_STATUS (private_status,
198 "Cannot find document for doc_id from query");
202 _notmuch_message_remove_term (message, "thread", loser_thread_id);
203 _notmuch_message_add_term (message, "thread", winner_thread_id);
204 _notmuch_message_sync (message);
206 notmuch_message_destroy (message);
212 notmuch_message_destroy (message);
218 _my_talloc_free_for_g_hash (void *ptr)
224 _notmuch_database_link_message_to_parents (notmuch_database_t *notmuch,
225 notmuch_message_t *message,
226 notmuch_message_file_t *message_file,
227 const char **thread_id)
229 GHashTable *parents = NULL;
230 const char *refs, *in_reply_to, *in_reply_to_message_id, *strict_message_id = NULL;
231 const char *last_ref_message_id, *this_message_id;
232 GList *l, *keys = NULL;
233 notmuch_status_t ret = NOTMUCH_STATUS_SUCCESS;
235 parents = g_hash_table_new_full (g_str_hash, g_str_equal,
236 _my_talloc_free_for_g_hash, NULL);
237 this_message_id = notmuch_message_get_message_id (message);
239 refs = _notmuch_message_file_get_header (message_file, "references");
240 last_ref_message_id = parse_references (message,
244 in_reply_to = _notmuch_message_file_get_header (message_file, "in-reply-to");
246 strict_message_id = _notmuch_message_id_parse_strict (message,
249 in_reply_to_message_id = parse_references (message,
251 parents, in_reply_to);
253 /* For the parent of this message, use
254 * 1) the In-Reply-To header, if it looks sane, otherwise
255 * 2) the last message ID of the References header, if available.
256 * 3) Otherwise, fall back to the first message ID in
257 * the In-Reply-To header.
260 if (strict_message_id) {
261 _notmuch_message_add_term (message, "replyto", strict_message_id);
262 } else if (last_ref_message_id) {
263 _notmuch_message_add_term (message, "replyto",
264 last_ref_message_id);
265 } else if (in_reply_to_message_id) {
266 _notmuch_message_add_term (message, "replyto",
267 in_reply_to_message_id);
270 keys = g_hash_table_get_keys (parents);
271 for (l = keys; l; l = l->next) {
272 char *parent_message_id;
273 const char *parent_thread_id = NULL;
275 parent_message_id = (char *) l->data;
277 _notmuch_message_add_term (message, "reference",
280 ret = _resolve_message_id_to_thread_id (notmuch,
287 if (*thread_id == NULL) {
288 *thread_id = talloc_strdup (message, parent_thread_id);
289 _notmuch_message_add_term (message, "thread", *thread_id);
290 } else if (strcmp (*thread_id, parent_thread_id)) {
291 ret = _merge_threads (notmuch, *thread_id, parent_thread_id);
301 g_hash_table_unref (parents);
306 static notmuch_status_t
307 _notmuch_database_link_message_to_children (notmuch_database_t *notmuch,
308 notmuch_message_t *message,
309 const char **thread_id)
311 const char *message_id = notmuch_message_get_message_id (message);
312 Xapian::PostingIterator child, children_end;
313 notmuch_message_t *child_message = NULL;
314 const char *child_thread_id;
315 notmuch_status_t ret = NOTMUCH_STATUS_SUCCESS;
316 notmuch_private_status_t private_status;
318 _notmuch_database_find_doc_ids (notmuch, "reference", message_id, &child, &children_end);
320 for ( ; child != children_end; child++) {
322 child_message = _notmuch_message_create (message, notmuch,
323 *child, &private_status);
324 if (child_message == NULL) {
325 ret = COERCE_STATUS (private_status,
326 "Cannot find document for doc_id from query");
330 child_thread_id = notmuch_message_get_thread_id (child_message);
331 if (*thread_id == NULL) {
332 *thread_id = talloc_strdup (message, child_thread_id);
333 _notmuch_message_add_term (message, "thread", *thread_id);
334 } else if (strcmp (*thread_id, child_thread_id)) {
335 _notmuch_message_remove_term (child_message, "reference",
337 _notmuch_message_sync (child_message);
338 ret = _merge_threads (notmuch, *thread_id, child_thread_id);
343 notmuch_message_destroy (child_message);
344 child_message = NULL;
349 notmuch_message_destroy (child_message);
354 /* Fetch and clear the stored thread_id for message, or NULL if none. */
356 _consume_metadata_thread_id (void *ctx, notmuch_database_t *notmuch,
357 notmuch_message_t *message)
359 const char *message_id;
360 std::string stored_id;
363 message_id = notmuch_message_get_message_id (message);
364 metadata_key = _get_metadata_thread_id_key (ctx, message_id);
366 /* Check if we have already seen related messages to this one.
367 * If we have then use the thread_id that we stored at that time.
369 stored_id = notmuch->xapian_db->get_metadata (metadata_key);
370 if (stored_id.empty ()) {
373 Xapian::WritableDatabase *db;
375 db = static_cast <Xapian::WritableDatabase *> (notmuch->xapian_db);
377 /* Clear the metadata for this message ID. We don't need it
379 db->set_metadata (metadata_key, "");
381 return talloc_strdup (ctx, stored_id.c_str ());
385 /* Given a blank or ghost 'message' and its corresponding
386 * 'message_file' link it to existing threads in the database.
388 * First, if is_ghost, this retrieves the thread ID already stored in
389 * the message (which will be the case if a message was previously
390 * added that referenced this one). If the message is blank
391 * (!is_ghost), it doesn't have a thread ID yet (we'll generate one
392 * later in this function). If the database does not support ghost
393 * messages, this checks for a thread ID stored in database metadata
394 * for this message ID.
396 * Second, we look at 'message_file' and its link-relevant headers
397 * (References and In-Reply-To) for message IDs.
399 * Finally, we look in the database for existing message that
400 * reference 'message'.
402 * In all cases, we assign to the current message the first thread ID
403 * found. We will also merge any existing, distinct threads where this
404 * message belongs to both, (which is not uncommon when messages are
405 * processed out of order).
407 * Finally, if no thread ID has been found through referenced messages, we
408 * call _notmuch_message_generate_thread_id to generate a new thread
409 * ID. This should only happen for new, top-level messages, (no
410 * References or In-Reply-To header in this message, and no previously
411 * added message refers to this message).
413 static notmuch_status_t
414 _notmuch_database_link_message (notmuch_database_t *notmuch,
415 notmuch_message_t *message,
416 notmuch_message_file_t *message_file,
419 void *local = talloc_new (NULL);
420 notmuch_status_t status;
421 const char *thread_id = NULL;
423 /* Check if the message already had a thread ID */
424 if (notmuch->features & NOTMUCH_FEATURE_GHOSTS) {
426 thread_id = notmuch_message_get_thread_id (message);
428 thread_id = _consume_metadata_thread_id (local, notmuch, message);
430 _notmuch_message_add_term (message, "thread", thread_id);
433 status = _notmuch_database_link_message_to_parents (notmuch, message,
439 if (! (notmuch->features & NOTMUCH_FEATURE_GHOSTS)) {
440 /* In general, it shouldn't be necessary to link children,
441 * since the earlier indexing of those children will have
442 * stored a thread ID for the missing parent. However, prior
443 * to ghost messages, these stored thread IDs were NOT
444 * rewritten during thread merging (and there was no
445 * performant way to do so), so if indexed children were
446 * pulled into a different thread ID by a merge, it was
447 * necessary to pull them *back* into the stored thread ID of
448 * the parent. With ghost messages, we just rewrite the
449 * stored thread IDs during merging, so this workaround isn't
451 status = _notmuch_database_link_message_to_children (notmuch, message,
457 /* If not part of any existing thread, generate a new thread ID. */
458 if (thread_id == NULL) {
459 thread_id = _notmuch_database_generate_thread_id (notmuch);
461 _notmuch_message_add_term (message, "thread", thread_id);
471 notmuch_database_index_file (notmuch_database_t *notmuch,
472 const char *filename,
473 notmuch_indexopts_t *indexopts,
474 notmuch_message_t **message_ret)
476 notmuch_message_file_t *message_file;
477 notmuch_message_t *message = NULL;
478 notmuch_status_t ret = NOTMUCH_STATUS_SUCCESS, ret2;
479 notmuch_private_status_t private_status;
480 bool is_ghost = false, is_new = false;
481 notmuch_indexopts_t *def_indexopts = NULL;
484 const char *from, *to, *subject;
485 char *message_id = NULL;
490 ret = _notmuch_database_ensure_writable (notmuch);
494 message_file = _notmuch_message_file_open (notmuch, filename);
495 if (message_file == NULL)
496 return NOTMUCH_STATUS_FILE_ERROR;
498 /* Adding a message may change many documents. Do this all
500 ret = notmuch_database_begin_atomic (notmuch);
504 ret = _notmuch_message_file_get_headers (message_file,
505 &from, &subject, &to, &date,
511 /* Now that we have a message ID, we get a message object,
512 * (which may or may not reference an existing document in the
515 message = _notmuch_message_create_for_message_id (notmuch,
519 talloc_free (message_id);
521 /* We cannot call notmuch_message_get_flag for a new message */
522 switch (private_status) {
523 case NOTMUCH_PRIVATE_STATUS_NO_DOCUMENT_FOUND:
527 case NOTMUCH_PRIVATE_STATUS_SUCCESS:
528 is_ghost = notmuch_message_get_flag (message, NOTMUCH_MESSAGE_FLAG_GHOST);
532 ret = COERCE_STATUS (private_status,
533 "Unexpected status value from _notmuch_message_create_for_message_id");
537 _notmuch_message_add_filename (message, filename);
539 if (is_new || is_ghost) {
540 _notmuch_message_add_term (message, "type", "mail");
542 /* Convert ghost message to a regular message */
543 _notmuch_message_remove_term (message, "type", "ghost");
546 ret = _notmuch_database_link_message (notmuch, message,
547 message_file, is_ghost);
551 if (is_new || is_ghost)
552 _notmuch_message_set_header_values (message, date, from, subject);
555 def_indexopts = notmuch_database_get_default_indexopts (notmuch);
556 indexopts = def_indexopts;
559 ret = _notmuch_message_index_file (message, indexopts, message_file);
563 if (! is_new && !is_ghost)
564 ret = NOTMUCH_STATUS_DUPLICATE_MESSAGE_ID;
566 _notmuch_message_sync (message);
567 } catch (const Xapian::Error &error) {
568 _notmuch_database_log (notmuch, "A Xapian exception occurred adding message: %s.\n",
569 error.get_msg().c_str());
570 notmuch->exception_reported = true;
571 ret = NOTMUCH_STATUS_XAPIAN_EXCEPTION;
577 notmuch_indexopts_destroy (def_indexopts);
580 if ((ret == NOTMUCH_STATUS_SUCCESS ||
581 ret == NOTMUCH_STATUS_DUPLICATE_MESSAGE_ID) && message_ret)
582 *message_ret = message;
584 notmuch_message_destroy (message);
588 _notmuch_message_file_close (message_file);
590 ret2 = notmuch_database_end_atomic (notmuch);
591 if ((ret == NOTMUCH_STATUS_SUCCESS ||
592 ret == NOTMUCH_STATUS_DUPLICATE_MESSAGE_ID) &&
593 ret2 != NOTMUCH_STATUS_SUCCESS)
600 notmuch_database_add_message (notmuch_database_t *notmuch,
601 const char *filename,
602 notmuch_message_t **message_ret)
604 return notmuch_database_index_file (notmuch, filename,