1 /* query.cc - Support for searching a notmuch database
3 * Copyright © 2009 Carl Worth
5 * This program is free software: you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation, either version 3 of the License, or
8 * (at your option) any later version.
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
15 * You should have received a copy of the GNU General Public License
16 * along with this program. If not, see https://www.gnu.org/licenses/ .
18 * Author: Carl Worth <cworth@cworth.org>
21 #include "notmuch-private.h"
22 #include "database-private.h"
24 #include <glib.h> /* GHashTable, GPtrArray */
26 struct _notmuch_query {
27 notmuch_database_t *notmuch;
28 const char *query_string;
30 notmuch_string_list_t *exclude_terms;
31 notmuch_exclude_t omit_excluded;
32 notmuch_bool_t parsed;
33 Xapian::Query xapian_query;
36 typedef struct _notmuch_mset_messages {
37 notmuch_messages_t base;
38 notmuch_database_t *notmuch;
39 Xapian::MSetIterator iterator;
40 Xapian::MSetIterator iterator_end;
41 } notmuch_mset_messages_t;
43 struct _notmuch_doc_id_set {
44 unsigned char *bitmap;
48 #define DOCIDSET_WORD(bit) ((bit) / CHAR_BIT)
49 #define DOCIDSET_BIT(bit) ((bit) % CHAR_BIT)
51 struct visible _notmuch_threads {
52 notmuch_query_t *query;
54 /* The ordered list of doc ids matched by the query. */
56 /* Our iterator's current position in doc_ids. */
57 unsigned int doc_id_pos;
58 /* The set of matched docid's that have not been assigned to a
59 * thread. Initially, this contains every docid in doc_ids. */
60 notmuch_doc_id_set_t match_set;
63 /* We need this in the message functions so forward declare. */
65 _notmuch_doc_id_set_init (void *ctx,
66 notmuch_doc_id_set_t *doc_ids,
72 char *env = getenv ("NOTMUCH_DEBUG_QUERY");
73 return (env && strcmp (env, "") != 0);
76 /* Explicit destructor call for placement new */
78 _notmuch_query_destructor (notmuch_query_t *query) {
79 query->xapian_query.~Query();
84 notmuch_query_create (notmuch_database_t *notmuch,
85 const char *query_string)
87 notmuch_query_t *query;
90 fprintf (stderr, "Query string is:\n%s\n", query_string);
92 query = talloc (notmuch, notmuch_query_t);
93 if (unlikely (query == NULL))
96 new (&query->xapian_query) Xapian::Query ();
97 query->parsed = FALSE;
99 talloc_set_destructor (query, _notmuch_query_destructor);
101 query->notmuch = notmuch;
103 query->query_string = talloc_strdup (query, query_string);
105 query->sort = NOTMUCH_SORT_NEWEST_FIRST;
107 query->exclude_terms = _notmuch_string_list_create (query);
109 query->omit_excluded = NOTMUCH_EXCLUDE_TRUE;
114 static notmuch_status_t
115 _notmuch_query_ensure_parsed (notmuch_query_t *query)
118 return NOTMUCH_STATUS_SUCCESS;
121 query->xapian_query =
122 query->notmuch->query_parser->
123 parse_query (query->query_string, NOTMUCH_QUERY_PARSER_FLAGS);
125 query->parsed = TRUE;
127 } catch (const Xapian::Error &error) {
128 if (!query->notmuch->exception_reported) {
129 _notmuch_database_log (query->notmuch,
130 "A Xapian exception occurred parsing query: %s\n",
131 error.get_msg ().c_str ());
132 _notmuch_database_log_append (query->notmuch,
133 "Query string was: %s\n",
134 query->query_string);
135 query->notmuch->exception_reported = TRUE;
138 return NOTMUCH_STATUS_XAPIAN_EXCEPTION;
140 return NOTMUCH_STATUS_SUCCESS;
144 notmuch_query_get_query_string (const notmuch_query_t *query)
146 return query->query_string;
150 notmuch_query_set_omit_excluded (notmuch_query_t *query,
151 notmuch_exclude_t omit_excluded)
153 query->omit_excluded = omit_excluded;
157 notmuch_query_set_sort (notmuch_query_t *query, notmuch_sort_t sort)
163 notmuch_query_get_sort (const notmuch_query_t *query)
169 notmuch_query_add_tag_exclude (notmuch_query_t *query, const char *tag)
171 char *term = talloc_asprintf (query, "%s%s", _find_prefix ("tag"), tag);
172 _notmuch_string_list_append (query->exclude_terms, term);
175 /* We end up having to call the destructors explicitly because we had
176 * to use "placement new" in order to initialize C++ objects within a
177 * block that we allocated with talloc. So C++ is making talloc
178 * slightly less simple to use, (we wouldn't need
179 * talloc_set_destructor at all otherwise).
182 _notmuch_messages_destructor (notmuch_mset_messages_t *messages)
184 messages->iterator.~MSetIterator ();
185 messages->iterator_end.~MSetIterator ();
190 /* Return a query that matches messages with the excluded tags
191 * registered with query. Any tags that explicitly appear in xquery
192 * will not be excluded, and will be removed from the list of exclude
193 * tags. The caller of this function has to combine the returned
194 * query appropriately.*/
196 _notmuch_exclude_tags (notmuch_query_t *query, Xapian::Query xquery)
198 Xapian::Query exclude_query = Xapian::Query::MatchNothing;
200 for (notmuch_string_node_t *term = query->exclude_terms->head; term;
202 Xapian::TermIterator it = xquery.get_terms_begin ();
203 Xapian::TermIterator end = xquery.get_terms_end ();
204 for (; it != end; it++) {
205 if ((*it).compare (term->string) == 0)
209 exclude_query = Xapian::Query (Xapian::Query::OP_OR,
210 exclude_query, Xapian::Query (term->string));
212 term->string = talloc_strdup (query, "");
214 return exclude_query;
218 notmuch_query_search_messages (notmuch_query_t *query)
220 notmuch_status_t status;
221 notmuch_messages_t *messages;
222 status = notmuch_query_search_messages_st (query, &messages);
230 notmuch_query_search_messages_st (notmuch_query_t *query,
231 notmuch_messages_t **out)
233 return _notmuch_query_search_documents (query, "mail", out);
237 _notmuch_query_search_documents (notmuch_query_t *query,
239 notmuch_messages_t **out)
241 notmuch_database_t *notmuch = query->notmuch;
242 const char *query_string = query->query_string;
243 notmuch_mset_messages_t *messages;
244 notmuch_status_t status;
246 status = _notmuch_query_ensure_parsed (query);
250 messages = talloc (query, notmuch_mset_messages_t);
251 if (unlikely (messages == NULL))
252 return NOTMUCH_STATUS_OUT_OF_MEMORY;
256 messages->base.is_of_list_type = FALSE;
257 messages->base.iterator = NULL;
258 messages->notmuch = notmuch;
259 new (&messages->iterator) Xapian::MSetIterator ();
260 new (&messages->iterator_end) Xapian::MSetIterator ();
262 talloc_set_destructor (messages, _notmuch_messages_destructor);
264 Xapian::Enquire enquire (*notmuch->xapian_db);
265 Xapian::Query mail_query (talloc_asprintf (query, "%s%s",
266 _find_prefix ("type"),
268 Xapian::Query final_query, exclude_query;
270 Xapian::MSetIterator iterator;
272 if (strcmp (query_string, "") == 0 ||
273 strcmp (query_string, "*") == 0)
275 final_query = mail_query;
277 final_query = Xapian::Query (Xapian::Query::OP_AND,
278 mail_query, query->xapian_query);
280 messages->base.excluded_doc_ids = NULL;
282 if ((query->omit_excluded != NOTMUCH_EXCLUDE_FALSE) && (query->exclude_terms)) {
283 exclude_query = _notmuch_exclude_tags (query, final_query);
285 if (query->omit_excluded == NOTMUCH_EXCLUDE_TRUE ||
286 query->omit_excluded == NOTMUCH_EXCLUDE_ALL)
288 final_query = Xapian::Query (Xapian::Query::OP_AND_NOT,
289 final_query, exclude_query);
290 } else { /* NOTMUCH_EXCLUDE_FLAG */
291 exclude_query = Xapian::Query (Xapian::Query::OP_AND,
292 exclude_query, final_query);
294 enquire.set_weighting_scheme (Xapian::BoolWeight());
295 enquire.set_query (exclude_query);
297 mset = enquire.get_mset (0, notmuch->xapian_db->get_doccount ());
299 GArray *excluded_doc_ids = g_array_new (FALSE, FALSE, sizeof (unsigned int));
301 for (iterator = mset.begin (); iterator != mset.end (); iterator++) {
302 unsigned int doc_id = *iterator;
303 g_array_append_val (excluded_doc_ids, doc_id);
305 messages->base.excluded_doc_ids = talloc (messages, _notmuch_doc_id_set);
306 _notmuch_doc_id_set_init (query, messages->base.excluded_doc_ids,
308 g_array_unref (excluded_doc_ids);
313 enquire.set_weighting_scheme (Xapian::BoolWeight());
315 switch (query->sort) {
316 case NOTMUCH_SORT_OLDEST_FIRST:
317 enquire.set_sort_by_value (NOTMUCH_VALUE_TIMESTAMP, FALSE);
319 case NOTMUCH_SORT_NEWEST_FIRST:
320 enquire.set_sort_by_value (NOTMUCH_VALUE_TIMESTAMP, TRUE);
322 case NOTMUCH_SORT_MESSAGE_ID:
323 enquire.set_sort_by_value (NOTMUCH_VALUE_MESSAGE_ID, FALSE);
325 case NOTMUCH_SORT_UNSORTED:
329 if (_debug_query ()) {
330 fprintf (stderr, "Exclude query is:\n%s\n",
331 exclude_query.get_description ().c_str ());
332 fprintf (stderr, "Final query is:\n%s\n",
333 final_query.get_description ().c_str ());
336 enquire.set_query (final_query);
338 mset = enquire.get_mset (0, notmuch->xapian_db->get_doccount ());
340 messages->iterator = mset.begin ();
341 messages->iterator_end = mset.end ();
343 *out = &messages->base;
344 return NOTMUCH_STATUS_SUCCESS;
346 } catch (const Xapian::Error &error) {
347 _notmuch_database_log (notmuch,
348 "A Xapian exception occurred performing query: %s\n",
349 error.get_msg().c_str());
350 _notmuch_database_log_append (notmuch,
351 "Query string was: %s\n",
352 query->query_string);
354 notmuch->exception_reported = TRUE;
355 talloc_free (messages);
356 return NOTMUCH_STATUS_XAPIAN_EXCEPTION;
361 _notmuch_mset_messages_valid (notmuch_messages_t *messages)
363 notmuch_mset_messages_t *mset_messages;
365 mset_messages = (notmuch_mset_messages_t *) messages;
367 return (mset_messages->iterator != mset_messages->iterator_end);
371 _notmuch_mset_messages_get_doc_id (notmuch_messages_t *messages)
373 notmuch_mset_messages_t *mset_messages;
375 mset_messages = (notmuch_mset_messages_t *) messages;
377 if (! _notmuch_mset_messages_valid (&mset_messages->base))
380 return *mset_messages->iterator;
384 _notmuch_mset_messages_get (notmuch_messages_t *messages)
386 notmuch_message_t *message;
387 Xapian::docid doc_id;
388 notmuch_private_status_t status;
389 notmuch_mset_messages_t *mset_messages;
391 mset_messages = (notmuch_mset_messages_t *) messages;
393 if (! _notmuch_mset_messages_valid (&mset_messages->base))
396 doc_id = *mset_messages->iterator;
398 message = _notmuch_message_create (mset_messages,
399 mset_messages->notmuch, doc_id,
402 if (message == NULL &&
403 status == NOTMUCH_PRIVATE_STATUS_NO_DOCUMENT_FOUND)
405 INTERNAL_ERROR ("a messages iterator contains a non-existent document ID.\n");
408 if (messages->excluded_doc_ids &&
409 _notmuch_doc_id_set_contains (messages->excluded_doc_ids, doc_id))
410 notmuch_message_set_flag (message, NOTMUCH_MESSAGE_FLAG_EXCLUDED, TRUE);
416 _notmuch_mset_messages_move_to_next (notmuch_messages_t *messages)
418 notmuch_mset_messages_t *mset_messages;
420 mset_messages = (notmuch_mset_messages_t *) messages;
422 mset_messages->iterator++;
425 static notmuch_bool_t
426 _notmuch_doc_id_set_init (void *ctx,
427 notmuch_doc_id_set_t *doc_ids,
430 unsigned int max = 0;
431 unsigned char *bitmap;
433 for (unsigned int i = 0; i < arr->len; i++)
434 max = MAX(max, g_array_index (arr, unsigned int, i));
435 bitmap = talloc_zero_array (ctx, unsigned char, DOCIDSET_WORD(max) + 1);
440 doc_ids->bitmap = bitmap;
441 doc_ids->bound = max + 1;
443 for (unsigned int i = 0; i < arr->len; i++) {
444 unsigned int doc_id = g_array_index (arr, unsigned int, i);
445 bitmap[DOCIDSET_WORD(doc_id)] |= 1 << DOCIDSET_BIT(doc_id);
452 _notmuch_doc_id_set_contains (notmuch_doc_id_set_t *doc_ids,
455 if (doc_id >= doc_ids->bound)
457 return doc_ids->bitmap[DOCIDSET_WORD(doc_id)] & (1 << DOCIDSET_BIT(doc_id));
461 _notmuch_doc_id_set_remove (notmuch_doc_id_set_t *doc_ids,
464 if (doc_id < doc_ids->bound)
465 doc_ids->bitmap[DOCIDSET_WORD(doc_id)] &= ~(1 << DOCIDSET_BIT(doc_id));
468 /* Glib objects force use to use a talloc destructor as well, (but not
469 * nearly as ugly as the for messages due to C++ objects). At
470 * this point, I'd really like to have some talloc-friendly
471 * equivalents for the few pieces of glib that I'm using. */
473 _notmuch_threads_destructor (notmuch_threads_t *threads)
475 if (threads->doc_ids)
476 g_array_unref (threads->doc_ids);
483 notmuch_query_search_threads (notmuch_query_t *query)
485 notmuch_status_t status;
486 notmuch_threads_t *threads;
487 status = notmuch_query_search_threads_st (query, &threads);
495 notmuch_query_search_threads_st (notmuch_query_t *query,
496 notmuch_threads_t **out)
498 notmuch_threads_t *threads;
499 notmuch_messages_t *messages;
500 notmuch_status_t status;
502 threads = talloc (query, notmuch_threads_t);
504 return NOTMUCH_STATUS_OUT_OF_MEMORY;
505 threads->doc_ids = NULL;
506 talloc_set_destructor (threads, _notmuch_threads_destructor);
508 threads->query = query;
510 status = notmuch_query_search_messages_st (query, &messages);
512 talloc_free (threads);
516 threads->doc_ids = g_array_new (FALSE, FALSE, sizeof (unsigned int));
517 while (notmuch_messages_valid (messages)) {
518 unsigned int doc_id = _notmuch_mset_messages_get_doc_id (messages);
519 g_array_append_val (threads->doc_ids, doc_id);
520 notmuch_messages_move_to_next (messages);
522 threads->doc_id_pos = 0;
524 talloc_free (messages);
526 if (! _notmuch_doc_id_set_init (threads, &threads->match_set,
528 talloc_free (threads);
529 return NOTMUCH_STATUS_OUT_OF_MEMORY;
533 return NOTMUCH_STATUS_SUCCESS;
537 notmuch_query_destroy (notmuch_query_t *query)
543 notmuch_threads_valid (notmuch_threads_t *threads)
550 while (threads->doc_id_pos < threads->doc_ids->len) {
551 doc_id = g_array_index (threads->doc_ids, unsigned int,
552 threads->doc_id_pos);
553 if (_notmuch_doc_id_set_contains (&threads->match_set, doc_id))
556 threads->doc_id_pos++;
559 return threads->doc_id_pos < threads->doc_ids->len;
563 notmuch_threads_get (notmuch_threads_t *threads)
567 if (! notmuch_threads_valid (threads))
570 doc_id = g_array_index (threads->doc_ids, unsigned int,
571 threads->doc_id_pos);
572 return _notmuch_thread_create (threads->query,
573 threads->query->notmuch,
576 threads->query->exclude_terms,
577 threads->query->omit_excluded,
578 threads->query->sort);
582 notmuch_threads_move_to_next (notmuch_threads_t *threads)
584 threads->doc_id_pos++;
588 notmuch_threads_destroy (notmuch_threads_t *threads)
590 talloc_free (threads);
594 notmuch_query_count_messages (notmuch_query_t *query)
596 notmuch_status_t status;
599 status = notmuch_query_count_messages_st (query, &count);
600 return status ? 0 : count;
604 notmuch_query_count_messages_st (notmuch_query_t *query, unsigned *count_out)
606 return _notmuch_query_count_documents (query, "mail", count_out);
610 _notmuch_query_count_documents (notmuch_query_t *query, const char *type, unsigned *count_out)
612 notmuch_database_t *notmuch = query->notmuch;
613 const char *query_string = query->query_string;
614 Xapian::doccount count = 0;
615 notmuch_status_t status;
617 status = _notmuch_query_ensure_parsed (query);
622 Xapian::Enquire enquire (*notmuch->xapian_db);
623 Xapian::Query mail_query (talloc_asprintf (query, "%s%s",
624 _find_prefix ("type"),
626 Xapian::Query final_query, exclude_query;
629 if (strcmp (query_string, "") == 0 ||
630 strcmp (query_string, "*") == 0)
632 final_query = mail_query;
634 final_query = Xapian::Query (Xapian::Query::OP_AND,
635 mail_query, query->xapian_query);
638 exclude_query = _notmuch_exclude_tags (query, final_query);
640 final_query = Xapian::Query (Xapian::Query::OP_AND_NOT,
641 final_query, exclude_query);
643 enquire.set_weighting_scheme(Xapian::BoolWeight());
644 enquire.set_docid_order(Xapian::Enquire::ASCENDING);
646 if (_debug_query ()) {
647 fprintf (stderr, "Exclude query is:\n%s\n",
648 exclude_query.get_description ().c_str ());
649 fprintf (stderr, "Final query is:\n%s\n",
650 final_query.get_description ().c_str ());
653 enquire.set_query (final_query);
656 * Set the checkatleast parameter to the number of documents
657 * in the database to make get_matches_estimated() exact.
658 * Set the max parameter to 0 to avoid fetching documents we will discard.
660 mset = enquire.get_mset (0, 0,
661 notmuch->xapian_db->get_doccount ());
663 count = mset.get_matches_estimated();
665 } catch (const Xapian::Error &error) {
666 _notmuch_database_log (notmuch,
667 "A Xapian exception occurred performing query: %s\n",
668 error.get_msg().c_str());
669 _notmuch_database_log_append (notmuch,
670 "Query string was: %s\n",
671 query->query_string);
672 return NOTMUCH_STATUS_XAPIAN_EXCEPTION;
676 return NOTMUCH_STATUS_SUCCESS;
680 notmuch_query_count_threads (notmuch_query_t *query)
682 notmuch_status_t status;
685 status = notmuch_query_count_threads_st (query, &count);
686 return status ? 0 : count;
690 notmuch_query_count_threads_st (notmuch_query_t *query, unsigned *count)
692 notmuch_messages_t *messages;
695 notmuch_status_t ret = NOTMUCH_STATUS_SUCCESS;
698 query->sort = NOTMUCH_SORT_UNSORTED;
699 ret = notmuch_query_search_messages_st (query, &messages);
703 if (messages == NULL)
704 return NOTMUCH_STATUS_XAPIAN_EXCEPTION;
706 hash = g_hash_table_new_full (g_str_hash, g_str_equal, NULL, NULL);
708 talloc_free (messages);
709 return NOTMUCH_STATUS_OUT_OF_MEMORY;
712 while (notmuch_messages_valid (messages)) {
713 notmuch_message_t *message = notmuch_messages_get (messages);
714 const char *thread_id = notmuch_message_get_thread_id (message);
715 char *thread_id_copy = talloc_strdup (messages, thread_id);
716 if (unlikely (thread_id_copy == NULL)) {
717 notmuch_message_destroy (message);
718 ret = NOTMUCH_STATUS_OUT_OF_MEMORY;
721 g_hash_table_insert (hash, thread_id_copy, NULL);
722 notmuch_message_destroy (message);
723 notmuch_messages_move_to_next (messages);
726 *count = g_hash_table_size (hash);
729 g_hash_table_unref (hash);
730 talloc_free (messages);
736 notmuch_query_get_database (const notmuch_query_t *query)
738 return query->notmuch;