1 /* notmuch - Not much of an email program, (just index and search)
3 * Copyright © 2009 Carl Worth
5 * This program is free software: you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation, either version 3 of the License, or
8 * (at your option) any later version.
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
15 * You should have received a copy of the GNU General Public License
16 * along with this program. If not, see http://www.gnu.org/licenses/ .
18 * Author: Carl Worth <cworth@cworth.org>
22 #define _GNU_SOURCE /* for getline */
28 /* This is separate from notmuch-private.h because we're trying to
29 * keep notmuch.c from looking into any internals, (which helps us
30 * develop notmuch.h into a plausible library interface).
44 #include <glib.h> /* g_strdup_printf */
46 /* There's no point in continuing when we've detected that we've done
47 * something wrong internally (as opposed to the user passing in a
50 * Note that __location__ comes from talloc.h.
52 #define INTERNAL_ERROR(format, ...) \
55 "Internal error: " format " (%s)\n", \
56 ##__VA_ARGS__, __location__); \
60 #define ARRAY_SIZE(arr) (sizeof (arr) / sizeof (arr[0]))
62 typedef int (*command_function_t) (int argc, char *argv[]);
64 typedef struct command {
66 command_function_t function;
71 int ignore_read_only_directories;
72 int saw_read_only_directory;
77 struct timeval tv_start;
81 chomp_newline (char *str)
83 if (str && str[strlen(str)-1] == '\n')
84 str[strlen(str)-1] = '\0';
87 /* Compute the number of seconds elapsed from start to end. */
89 tv_elapsed (struct timeval start, struct timeval end)
91 return ((end.tv_sec - start.tv_sec) +
92 (end.tv_usec - start.tv_usec) / 1e6);
96 print_formatted_seconds (double seconds)
102 printf ("almost no time");
106 if (seconds > 3600) {
107 hours = (int) seconds / 3600;
108 printf ("%dh ", hours);
109 seconds -= hours * 3600;
113 minutes = (int) seconds / 60;
114 printf ("%dm ", minutes);
115 seconds -= minutes * 60;
118 printf ("%ds", (int) seconds);
122 add_files_print_progress (add_files_state_t *state)
124 struct timeval tv_now;
125 double elapsed_overall, rate_overall;
127 gettimeofday (&tv_now, NULL);
129 elapsed_overall = tv_elapsed (state->tv_start, tv_now);
130 rate_overall = (state->processed_files) / elapsed_overall;
132 printf ("Processed %d", state->processed_files);
134 if (state->total_files) {
135 printf (" of %d files (", state->total_files);
136 print_formatted_seconds ((state->total_files - state->processed_files) /
138 printf (" remaining). \r");
140 printf (" files (%d files/sec.) \r", (int) rate_overall);
146 /* Examine 'path' recursively as follows:
148 * o Ask the filesystem for the mtime of 'path' (path_mtime)
150 * o Ask the database for its timestamp of 'path' (path_dbtime)
152 * o If 'path_mtime' > 'path_dbtime'
154 * o For each regular file in 'path' with mtime newer than the
155 * 'path_dbtime' call add_message to add the file to the
158 * o For each sub-directory of path, recursively call into this
161 * o Tell the database to update its time of 'path' to 'path_mtime'
163 * The 'struct stat *st' must point to a structure that has already
164 * been initialized for 'path' by calling stat().
167 add_files_recursive (notmuch_database_t *notmuch,
170 add_files_state_t *state)
173 struct dirent *e, *entry = NULL;
177 time_t path_mtime, path_dbtime;
178 notmuch_status_t status, ret = NOTMUCH_STATUS_SUCCESS;
180 /* If we're told to, we bail out on encountering a read-only
181 * directory, (with this being a clear clue from the user to
182 * Notmuch that new mail won't be arriving there and we need not
184 if (state->ignore_read_only_directories &&
185 (st->st_mode & S_IWUSR) == 0)
187 state->saw_read_only_directory = TRUE;
191 path_mtime = st->st_mtime;
193 path_dbtime = notmuch_database_get_timestamp (notmuch, path);
195 dir = opendir (path);
197 fprintf (stderr, "Error opening directory %s: %s\n",
198 path, strerror (errno));
199 ret = NOTMUCH_STATUS_FILE_ERROR;
203 entry_length = offsetof (struct dirent, d_name) +
204 pathconf (path, _PC_NAME_MAX) + 1;
205 entry = malloc (entry_length);
208 err = readdir_r (dir, entry, &e);
210 fprintf (stderr, "Error reading directory: %s\n",
212 ret = NOTMUCH_STATUS_FILE_ERROR;
219 /* If this directory hasn't been modified since the last
220 * add_files, then we only need to look further for
221 * sub-directories. */
222 if (path_mtime <= path_dbtime && entry->d_type != DT_DIR)
225 /* Ignore special directories to avoid infinite recursion.
226 * Also ignore the .notmuch directory.
228 /* XXX: Eventually we'll want more sophistication to let the
229 * user specify files to be ignored. */
230 if (strcmp (entry->d_name, ".") == 0 ||
231 strcmp (entry->d_name, "..") == 0 ||
232 strcmp (entry->d_name, ".notmuch") ==0)
237 next = g_strdup_printf ("%s/%s", path, entry->d_name);
239 if (stat (next, st)) {
240 fprintf (stderr, "Error reading %s: %s\n",
241 next, strerror (errno));
242 ret = NOTMUCH_STATUS_FILE_ERROR;
246 if (S_ISREG (st->st_mode)) {
247 /* If the file hasn't been modified since the last
248 * add_files, then we need not look at it. */
249 if (st->st_mtime > path_dbtime) {
250 state->processed_files++;
252 status = notmuch_database_add_message (notmuch, next);
255 case NOTMUCH_STATUS_SUCCESS:
256 state->added_messages++;
258 /* Non-fatal issues (go on to next file) */
259 case NOTMUCH_STATUS_DUPLICATE_MESSAGE_ID:
260 /* Stay silent on this one. */
262 case NOTMUCH_STATUS_FILE_NOT_EMAIL:
263 fprintf (stderr, "Note: Ignoring non-mail file: %s\n",
266 /* Fatal issues. Don't process anymore. */
267 case NOTMUCH_STATUS_XAPIAN_EXCEPTION:
268 fprintf (stderr, "A Xapian error was encountered. Halting processing.\n");
272 INTERNAL_ERROR ("add_message returned unexpected value: %d", status);
275 if (state->processed_files % 1000 == 0)
276 add_files_print_progress (state);
278 } else if (S_ISDIR (st->st_mode)) {
279 status = add_files_recursive (notmuch, next, st, state);
280 if (status && ret == NOTMUCH_STATUS_SUCCESS)
288 status = notmuch_database_set_timestamp (notmuch, path, path_mtime);
289 if (status && ret == NOTMUCH_STATUS_SUCCESS)
303 /* This is the top-level entry point for add_files. It does a couple
304 * of error checks, and then calls into the recursive function,
305 * (avoiding the repeating of these error checks at every
306 * level---which would be useless becaues we already do a stat() at
307 * the level above). */
308 static notmuch_status_t
309 add_files (notmuch_database_t *notmuch,
311 add_files_state_t *state)
315 if (stat (path, &st)) {
316 fprintf (stderr, "Error reading directory %s: %s\n",
317 path, strerror (errno));
318 return NOTMUCH_STATUS_FILE_ERROR;
321 if (! S_ISDIR (st.st_mode)) {
322 fprintf (stderr, "Error: %s is not a directory.\n", path);
323 return NOTMUCH_STATUS_FILE_ERROR;
326 return add_files_recursive (notmuch, path, &st, state);
329 /* Recursively count all regular files in path and all sub-direcotries
330 * of path. The result is added to *count (which should be
331 * initialized to zero by the top-level caller before calling
334 count_files (const char *path, int *count)
337 struct dirent *entry, *e;
343 dir = opendir (path);
346 fprintf (stderr, "Warning: failed to open directory %s: %s\n",
347 path, strerror (errno));
351 entry_length = offsetof (struct dirent, d_name) +
352 pathconf (path, _PC_NAME_MAX) + 1;
353 entry = malloc (entry_length);
356 err = readdir_r (dir, entry, &e);
358 fprintf (stderr, "Error reading directory: %s\n",
367 /* Ignore special directories to avoid infinite recursion.
368 * Also ignore the .notmuch directory.
370 /* XXX: Eventually we'll want more sophistication to let the
371 * user specify files to be ignored. */
372 if (strcmp (entry->d_name, ".") == 0 ||
373 strcmp (entry->d_name, "..") == 0 ||
374 strcmp (entry->d_name, ".notmuch") == 0)
379 next = g_strdup_printf ("%s/%s", path, entry->d_name);
383 if (S_ISREG (st.st_mode)) {
385 if (*count % 1000 == 0) {
386 printf ("Found %d files so far.\r", *count);
389 } else if (S_ISDIR (st.st_mode)) {
390 count_files (next, count);
402 setup_command (int argc, char *argv[])
404 notmuch_database_t *notmuch = NULL;
405 char *default_path, *mail_directory = NULL;
408 add_files_state_t add_files_state;
410 struct timeval tv_now;
411 notmuch_status_t ret = NOTMUCH_STATUS_SUCCESS;
413 printf ("Welcome to notmuch!\n\n");
415 printf ("The goal of notmuch is to help you manage and search your collection of\n"
416 "email, and to efficiently keep up with the flow of email as it comes in.\n\n");
418 printf ("Notmuch needs to know the top-level directory of your email archive,\n"
419 "(where you already have mail stored and where messages will be delivered\n"
420 "in the future). This directory can contain any number of sub-directories\n"
421 "and primarily just files with indvidual email messages (eg. maildir or mh\n"
422 "archives are perfect). If there are other, non-email files (such as\n"
423 "indexes maintained by other email programs) then notmuch will do its\n"
424 "best to detect those and ignore them.\n\n");
426 printf ("Mail storage that uses mbox format, (where one mbox file contains many\n"
427 "messages), will not work with notmuch. If that's how your mail is currently\n"
428 "stored, we recommend you first convert it to maildir format with a utility\n"
429 "such as mb2md. In that case, press Control-C now and run notmuch again\n"
430 "once the conversion is complete.\n\n");
433 default_path = notmuch_database_default_path ();
434 printf ("Top-level mail directory [%s]: ", default_path);
437 getline (&mail_directory, &line_size, stdin);
438 chomp_newline (mail_directory);
442 if (mail_directory == NULL || strlen (mail_directory) == 0) {
444 free (mail_directory);
445 mail_directory = default_path;
447 /* XXX: Instead of telling the user to use an environment
448 * variable here, we should really be writing out a configuration
449 * file and loading that on the next run. */
450 if (strcmp (mail_directory, default_path)) {
451 printf ("Note: Since you are not using the default path, you will want to set\n"
452 "the NOTMUCH_BASE environment variable to %s so that\n"
453 "future calls to notmuch commands will know where to find your mail.\n",
455 printf ("For example, if you are using bash for your shell, add:\n\n");
456 printf ("\texport NOTMUCH_BASE=%s\n\n", mail_directory);
457 printf ("to your ~/.bashrc file.\n\n");
462 notmuch = notmuch_database_create (mail_directory);
463 if (notmuch == NULL) {
464 fprintf (stderr, "Failed to create new notmuch database at %s\n",
466 ret = NOTMUCH_STATUS_FILE_ERROR;
470 printf ("OK. Let's take a look at the mail we can find in the directory\n");
471 printf ("%s ...\n", mail_directory);
474 count_files (mail_directory, &count);
476 printf ("Found %d total files. That's not much mail.\n\n", count);
478 printf ("Next, we'll inspect the messages and create a database of threads:\n");
480 add_files_state.ignore_read_only_directories = FALSE;
481 add_files_state.saw_read_only_directory = FALSE;
482 add_files_state.total_files = count;
483 add_files_state.processed_files = 0;
484 add_files_state.added_messages = 0;
485 gettimeofday (&add_files_state.tv_start, NULL);
487 ret = add_files (notmuch, mail_directory, &add_files_state);
489 gettimeofday (&tv_now, NULL);
490 elapsed = tv_elapsed (add_files_state.tv_start,
492 printf ("Processed %d %s in ", add_files_state.processed_files,
493 add_files_state.processed_files == 1 ?
494 "file" : "total files");
495 print_formatted_seconds (elapsed);
497 printf (" (%d files/sec.). \n",
498 (int) (add_files_state.processed_files / elapsed));
502 if (add_files_state.added_messages) {
503 printf ("Added %d %s to the database.\n\n",
504 add_files_state.added_messages,
505 add_files_state.added_messages == 1 ?
506 "message" : "unique messages");
509 printf ("When new mail is delivered to %s in the future,\n"
510 "run \"notmuch new\" to add it to the database.\n\n",
514 printf ("Note: At least one error was encountered: %s\n",
515 notmuch_status_to_string (ret));
520 free (mail_directory);
522 notmuch_database_close (notmuch);
528 new_command (int argc, char *argv[])
530 notmuch_database_t *notmuch;
531 const char *mail_directory;
532 add_files_state_t add_files_state;
534 struct timeval tv_now;
537 notmuch = notmuch_database_open (NULL);
538 if (notmuch == NULL) {
543 mail_directory = notmuch_database_get_path (notmuch);
545 add_files_state.ignore_read_only_directories = TRUE;
546 add_files_state.saw_read_only_directory = FALSE;
547 add_files_state.total_files = 0;
548 add_files_state.processed_files = 0;
549 add_files_state.added_messages = 0;
550 gettimeofday (&add_files_state.tv_start, NULL);
552 ret = add_files (notmuch, mail_directory, &add_files_state);
554 gettimeofday (&tv_now, NULL);
555 elapsed = tv_elapsed (add_files_state.tv_start,
557 if (add_files_state.processed_files) {
558 printf ("Processed %d %s in ", add_files_state.processed_files,
559 add_files_state.processed_files == 1 ?
560 "file" : "total files");
561 print_formatted_seconds (elapsed);
563 printf (" (%d files/sec.). \n",
564 (int) (add_files_state.processed_files / elapsed));
569 if (add_files_state.added_messages) {
570 printf ("Added %d new %s to the database (not much, really).\n",
571 add_files_state.added_messages,
572 add_files_state.added_messages == 1 ?
573 "message" : "messages");
575 printf ("No new mail---and that's not much.\n");
578 if (elapsed > 1 && ! add_files_state.saw_read_only_directory) {
579 printf ("\nTip: If you have any sub-directories that are archives (that is,\n"
580 "they will never receive new mail), marking these directores as\n"
581 "read-only (chmod u-w /path/to/dir) will make \"notmuch new\"\n"
582 "much more efficient (it won't even look in those directories).\n");
586 printf ("\nNote: At least one error was encountered: %s\n",
587 notmuch_status_to_string (ret));
592 notmuch_database_close (notmuch);
598 search_command (int argc, char *argv[])
600 void *local = talloc_new (NULL);
601 notmuch_database_t *notmuch = NULL;
602 notmuch_query_t *query;
603 notmuch_results_t *results;
604 notmuch_message_t *message;
605 notmuch_tags_t *tags;
608 notmuch_status_t ret = NOTMUCH_STATUS_SUCCESS;
610 notmuch = notmuch_database_open (NULL);
611 if (notmuch == NULL) {
616 /* XXX: Should add xtalloc wrappers here and use them. */
617 query_str = talloc_strdup (local, "");
619 for (i = 0; i < argc; i++) {
621 query_str = talloc_asprintf_append (query_str, " ");
623 query_str = talloc_asprintf_append (query_str, "%s", argv[i]);
626 query = notmuch_query_create (notmuch, query_str);
628 fprintf (stderr, "Out of memory\n");
633 for (results = notmuch_query_search (query);
634 notmuch_results_has_more (results);
635 notmuch_results_advance (results))
638 message = notmuch_results_get (results);
640 printf ("%s (", notmuch_message_get_message_id (message));
642 for (tags = notmuch_message_get_tags (message);
643 notmuch_tags_has_more (tags);
644 notmuch_tags_advance (tags))
649 printf ("%s", notmuch_tags_get (tags));
656 notmuch_message_destroy (message);
659 notmuch_query_destroy (query);
663 notmuch_database_close (notmuch);
670 show_command (int argc, char *argv[])
672 fprintf (stderr, "Error: show is not implemented yet.\n");
677 dump_command (int argc, char *argv[])
680 notmuch_database_t *notmuch = NULL;
681 notmuch_query_t *query;
682 notmuch_results_t *results;
683 notmuch_message_t *message;
684 notmuch_tags_t *tags;
688 output = fopen (argv[0], "w");
689 if (output == NULL) {
690 fprintf (stderr, "Error opening %s for writing: %s\n",
691 argv[0], strerror (errno));
699 notmuch = notmuch_database_open (NULL);
700 if (notmuch == NULL) {
705 query = notmuch_query_create (notmuch, "");
707 fprintf (stderr, "Out of memory\n");
712 notmuch_query_set_sort (query, NOTMUCH_SORT_MESSAGE_ID);
714 for (results = notmuch_query_search (query);
715 notmuch_results_has_more (results);
716 notmuch_results_advance (results))
719 message = notmuch_results_get (results);
722 "%s (", notmuch_message_get_message_id (message));
724 for (tags = notmuch_message_get_tags (message);
725 notmuch_tags_has_more (tags);
726 notmuch_tags_advance (tags))
729 fprintf (output, " ");
731 fprintf (output, "%s", notmuch_tags_get (tags));
736 fprintf (output, ")\n");
738 notmuch_message_destroy (message);
741 notmuch_query_destroy (query);
745 notmuch_database_close (notmuch);
746 if (output != stdout)
753 restore_command (int argc, char *argv[])
756 notmuch_database_t *notmuch = NULL;
765 input = fopen (argv[0], "r");
767 fprintf (stderr, "Error opening %s for reading: %s\n",
768 argv[0], strerror (errno));
773 printf ("No filename given. Reading dump from stdin.\n");
777 notmuch = notmuch_database_open (NULL);
778 if (notmuch == NULL) {
783 /* Dump output is one line per message. We match a sequence of
784 * non-space characters for the message-id, then one or more
785 * spaces, then a list of space-separated tags as a sequence of
786 * characters within literal '(' and ')'. */
788 "^([^ ]+) \\(([^)]*)\\)$",
791 while ((line_len = getline (&line, &line_size, input)) != -1) {
793 char *message_id, *tags, *tag, *next;
794 notmuch_message_t *message;
795 notmuch_status_t status;
797 chomp_newline (line);
799 rerr = xregexec (®ex, line, 3, match, 0);
800 if (rerr == REG_NOMATCH)
802 fprintf (stderr, "Warning: Ignoring invalid input line: %s\n",
807 message_id = xstrndup (line + match[1].rm_so,
808 match[1].rm_eo - match[1].rm_so);
809 tags = xstrndup (line + match[2].rm_so,
810 match[2].rm_eo - match[2].rm_so);
814 message = notmuch_database_find_message (notmuch, message_id);
815 if (message == NULL) {
816 fprintf (stderr, "Warning: Cannot apply tags to missing message: %s (",
822 tag = strsep (&next, " ");
826 status = notmuch_message_add_tag (message, tag);
829 "Error applying tag %s to message %s:\n",
831 fprintf (stderr, "%s\n",
832 notmuch_status_to_string (status));
835 fprintf (stderr, "%s%s",
836 tag == tags ? "" : " ", tag);
841 notmuch_message_destroy (message);
843 fprintf (stderr, ")\n");
855 notmuch_database_close (notmuch);
860 command_t commands[] = {
861 { "setup", setup_command,
862 "Interactively setup notmuch for first use.\n\n"
863 "\t\tInvoking notmuch with no command argument will run setup if\n"
864 "\t\tthe setup command has not previously been completed." },
865 { "new", new_command,
866 "Find and import any new messages.\n\n"
867 "\t\tScans all sub-directories of the database, adding new files\n"
868 "\t\tthat are found. Note: \"notmuch new\" will skip any\n"
869 "\t\tread-only directories, so you can use that to mark\n"
870 "\t\tdirectories that will not receive any new mail."},
871 { "search", search_command,
872 "<search-term> [...]\n\n"
873 "\t\tSearch for threads matching the given search terms.\n"
874 "\t\tOnce we actually implement search we'll document the\n"
875 "\t\tsyntax here." },
876 { "show", show_command,
878 "\t\tShow the thread with the given thread ID (see 'search')." },
879 { "dump", dump_command,
881 "\t\tCreate a plain-text dump of the tags for each message\n"
882 "\t\twriting to the given filename, if any, or to stdout.\n"
883 "\t\tThese tags are the only data in the notmuch database\n"
884 "\t\tthat can't be recreated from the messages themselves.\n"
885 "\t\tThe output of notmuch dump is therefore the only\n"
886 "\t\tcritical thing to backup (and much more friendly to\n"
887 "\t\tincremental backup than the native database files." },
888 { "restore", restore_command,
890 "\t\tRestore the tags from the given dump file (see 'dump')." }
899 fprintf (stderr, "Usage: notmuch <command> [args...]\n");
900 fprintf (stderr, "\n");
901 fprintf (stderr, "Where <command> and [args...] are as follows:\n");
902 fprintf (stderr, "\n");
904 for (i = 0; i < ARRAY_SIZE (commands); i++) {
905 command = &commands[i];
907 fprintf (stderr, "\t%s\t%s\n\n", command->name, command->usage);
912 main (int argc, char *argv[])
918 return setup_command (0, NULL);
920 for (i = 0; i < ARRAY_SIZE (commands); i++) {
921 command = &commands[i];
923 if (strcmp (argv[1], command->name) == 0)
924 return (command->function) (argc - 2, &argv[2]);
927 /* Don't complain about "help" being an unknown command when we're
928 about to provide exactly what's wanted anyway. */
929 if (strcmp (argv[1], "help") == 0 ||
930 strcmp (argv[1], "--help") == 0)
932 fprintf (stderr, "The notmuch mail system.\n\n");
934 fprintf (stderr, "Error: Unknown command '%s'\n\n", argv[1]);