1 /* notmuch - Not much of an email program, (just index and search)
3 * Copyright © 2009 Carl Worth
5 * This program is free software: you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation, either version 3 of the License, or
8 * (at your option) any later version.
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
15 * You should have received a copy of the GNU General Public License
16 * along with this program. If not, see http://www.gnu.org/licenses/ .
18 * Author: Carl Worth <cworth@cworth.org>
22 #define _GNU_SOURCE /* for getline */
28 /* This is separate from notmuch-private.h because we're trying to
29 * keep notmuch.c from looking into any internals, (which helps us
30 * develop notmuch.h into a plausible library interface).
44 #define unused(x) x __attribute__ ((unused))
46 /* There's no point in continuing when we've detected that we've done
47 * something wrong internally (as opposed to the user passing in a
50 * Note that __location__ comes from talloc.h.
52 #define INTERNAL_ERROR(format, ...) \
55 "Internal error: " format " (%s)\n", \
56 ##__VA_ARGS__, __location__); \
60 #define ARRAY_SIZE(arr) (sizeof (arr) / sizeof (arr[0]))
62 typedef int (*command_function_t) (int argc, char *argv[]);
64 typedef struct command {
66 command_function_t function;
71 int ignore_read_only_directories;
72 int saw_read_only_directory;
77 struct timeval tv_start;
81 chomp_newline (char *str)
83 if (str && str[strlen(str)-1] == '\n')
84 str[strlen(str)-1] = '\0';
87 /* Compute the number of seconds elapsed from start to end. */
89 tv_elapsed (struct timeval start, struct timeval end)
91 return ((end.tv_sec - start.tv_sec) +
92 (end.tv_usec - start.tv_usec) / 1e6);
96 print_formatted_seconds (double seconds)
102 printf ("almost no time");
106 if (seconds > 3600) {
107 hours = (int) seconds / 3600;
108 printf ("%dh ", hours);
109 seconds -= hours * 3600;
113 minutes = (int) seconds / 60;
114 printf ("%dm ", minutes);
115 seconds -= minutes * 60;
118 printf ("%ds", (int) seconds);
122 add_files_print_progress (add_files_state_t *state)
124 struct timeval tv_now;
125 double elapsed_overall, rate_overall;
127 gettimeofday (&tv_now, NULL);
129 elapsed_overall = tv_elapsed (state->tv_start, tv_now);
130 rate_overall = (state->processed_files) / elapsed_overall;
132 printf ("Processed %d", state->processed_files);
134 if (state->total_files) {
135 printf (" of %d files (", state->total_files);
136 print_formatted_seconds ((state->total_files - state->processed_files) /
138 printf (" remaining). \r");
140 printf (" files (%d files/sec.) \r", (int) rate_overall);
146 /* Examine 'path' recursively as follows:
148 * o Ask the filesystem for the mtime of 'path' (path_mtime)
150 * o Ask the database for its timestamp of 'path' (path_dbtime)
152 * o If 'path_mtime' > 'path_dbtime'
154 * o For each regular file in 'path' with mtime newer than the
155 * 'path_dbtime' call add_message to add the file to the
158 * o For each sub-directory of path, recursively call into this
161 * o Tell the database to update its time of 'path' to 'path_mtime'
163 * The 'struct stat *st' must point to a structure that has already
164 * been initialized for 'path' by calling stat().
166 static notmuch_status_t
167 add_files_recursive (notmuch_database_t *notmuch,
170 add_files_state_t *state)
173 struct dirent *e, *entry = NULL;
177 time_t path_mtime, path_dbtime;
178 notmuch_status_t status, ret = NOTMUCH_STATUS_SUCCESS;
180 /* If we're told to, we bail out on encountering a read-only
181 * directory, (with this being a clear clue from the user to
182 * Notmuch that new mail won't be arriving there and we need not
184 if (state->ignore_read_only_directories &&
185 (st->st_mode & S_IWUSR) == 0)
187 state->saw_read_only_directory = TRUE;
191 path_mtime = st->st_mtime;
193 path_dbtime = notmuch_database_get_timestamp (notmuch, path);
195 dir = opendir (path);
197 fprintf (stderr, "Error opening directory %s: %s\n",
198 path, strerror (errno));
199 ret = NOTMUCH_STATUS_FILE_ERROR;
203 entry_length = offsetof (struct dirent, d_name) +
204 pathconf (path, _PC_NAME_MAX) + 1;
205 entry = malloc (entry_length);
208 err = readdir_r (dir, entry, &e);
210 fprintf (stderr, "Error reading directory: %s\n",
212 ret = NOTMUCH_STATUS_FILE_ERROR;
219 /* If this directory hasn't been modified since the last
220 * add_files, then we only need to look further for
221 * sub-directories. */
222 if (path_mtime <= path_dbtime && entry->d_type != DT_DIR)
225 /* Ignore special directories to avoid infinite recursion.
226 * Also ignore the .notmuch directory.
228 /* XXX: Eventually we'll want more sophistication to let the
229 * user specify files to be ignored. */
230 if (strcmp (entry->d_name, ".") == 0 ||
231 strcmp (entry->d_name, "..") == 0 ||
232 strcmp (entry->d_name, ".notmuch") ==0)
237 next = talloc_asprintf (notmuch, "%s/%s", path, entry->d_name);
239 if (stat (next, st)) {
240 fprintf (stderr, "Error reading %s: %s\n",
241 next, strerror (errno));
242 ret = NOTMUCH_STATUS_FILE_ERROR;
246 if (S_ISREG (st->st_mode)) {
247 /* If the file hasn't been modified since the last
248 * add_files, then we need not look at it. */
249 if (st->st_mtime > path_dbtime) {
250 state->processed_files++;
252 status = notmuch_database_add_message (notmuch, next);
255 case NOTMUCH_STATUS_SUCCESS:
256 state->added_messages++;
258 /* Non-fatal issues (go on to next file) */
259 case NOTMUCH_STATUS_DUPLICATE_MESSAGE_ID:
260 /* Stay silent on this one. */
262 case NOTMUCH_STATUS_FILE_NOT_EMAIL:
263 fprintf (stderr, "Note: Ignoring non-mail file: %s\n",
266 /* Fatal issues. Don't process anymore. */
267 case NOTMUCH_STATUS_XAPIAN_EXCEPTION:
268 case NOTMUCH_STATUS_OUT_OF_MEMORY:
269 fprintf (stderr, "Error: %s. Halting processing.\n",
270 notmuch_status_to_string (status));
274 case NOTMUCH_STATUS_FILE_ERROR:
275 case NOTMUCH_STATUS_NULL_POINTER:
276 case NOTMUCH_STATUS_TAG_TOO_LONG:
277 case NOTMUCH_STATUS_LAST_STATUS:
278 INTERNAL_ERROR ("add_message returned unexpected value: %d", status);
281 if (state->processed_files % 1000 == 0)
282 add_files_print_progress (state);
284 } else if (S_ISDIR (st->st_mode)) {
285 status = add_files_recursive (notmuch, next, st, state);
286 if (status && ret == NOTMUCH_STATUS_SUCCESS)
294 status = notmuch_database_set_timestamp (notmuch, path, path_mtime);
295 if (status && ret == NOTMUCH_STATUS_SUCCESS)
309 /* This is the top-level entry point for add_files. It does a couple
310 * of error checks, and then calls into the recursive function,
311 * (avoiding the repeating of these error checks at every
312 * level---which would be useless becaues we already do a stat() at
313 * the level above). */
314 static notmuch_status_t
315 add_files (notmuch_database_t *notmuch,
317 add_files_state_t *state)
321 if (stat (path, &st)) {
322 fprintf (stderr, "Error reading directory %s: %s\n",
323 path, strerror (errno));
324 return NOTMUCH_STATUS_FILE_ERROR;
327 if (! S_ISDIR (st.st_mode)) {
328 fprintf (stderr, "Error: %s is not a directory.\n", path);
329 return NOTMUCH_STATUS_FILE_ERROR;
332 return add_files_recursive (notmuch, path, &st, state);
335 /* Recursively count all regular files in path and all sub-direcotries
336 * of path. The result is added to *count (which should be
337 * initialized to zero by the top-level caller before calling
340 count_files (const char *path, int *count)
343 struct dirent *e, *entry = NULL;
349 dir = opendir (path);
352 fprintf (stderr, "Warning: failed to open directory %s: %s\n",
353 path, strerror (errno));
357 entry_length = offsetof (struct dirent, d_name) +
358 pathconf (path, _PC_NAME_MAX) + 1;
359 entry = malloc (entry_length);
362 err = readdir_r (dir, entry, &e);
364 fprintf (stderr, "Error reading directory: %s\n",
373 /* Ignore special directories to avoid infinite recursion.
374 * Also ignore the .notmuch directory.
376 /* XXX: Eventually we'll want more sophistication to let the
377 * user specify files to be ignored. */
378 if (strcmp (entry->d_name, ".") == 0 ||
379 strcmp (entry->d_name, "..") == 0 ||
380 strcmp (entry->d_name, ".notmuch") == 0)
385 if (asprintf (&next, "%s/%s", path, entry->d_name) == -1) {
387 fprintf (stderr, "Error descending from %s to %s: Out of memory\n",
388 path, entry->d_name);
394 if (S_ISREG (st.st_mode)) {
396 if (*count % 1000 == 0) {
397 printf ("Found %d files so far.\r", *count);
400 } else if (S_ISDIR (st.st_mode)) {
401 count_files (next, count);
415 setup_command (unused (int argc), unused (char *argv[]))
417 notmuch_database_t *notmuch = NULL;
418 char *default_path, *mail_directory = NULL;
421 add_files_state_t add_files_state;
423 struct timeval tv_now;
424 notmuch_status_t ret = NOTMUCH_STATUS_SUCCESS;
426 printf ("Welcome to notmuch!\n\n");
428 printf ("The goal of notmuch is to help you manage and search your collection of\n"
429 "email, and to efficiently keep up with the flow of email as it comes in.\n\n");
431 printf ("Notmuch needs to know the top-level directory of your email archive,\n"
432 "(where you already have mail stored and where messages will be delivered\n"
433 "in the future). This directory can contain any number of sub-directories\n"
434 "and primarily just files with indvidual email messages (eg. maildir or mh\n"
435 "archives are perfect). If there are other, non-email files (such as\n"
436 "indexes maintained by other email programs) then notmuch will do its\n"
437 "best to detect those and ignore them.\n\n");
439 printf ("Mail storage that uses mbox format, (where one mbox file contains many\n"
440 "messages), will not work with notmuch. If that's how your mail is currently\n"
441 "stored, we recommend you first convert it to maildir format with a utility\n"
442 "such as mb2md. In that case, press Control-C now and run notmuch again\n"
443 "once the conversion is complete.\n\n");
446 default_path = notmuch_database_default_path ();
447 printf ("Top-level mail directory [%s]: ", default_path);
450 getline (&mail_directory, &line_size, stdin);
451 chomp_newline (mail_directory);
455 if (mail_directory == NULL || strlen (mail_directory) == 0) {
457 free (mail_directory);
458 mail_directory = default_path;
460 /* XXX: Instead of telling the user to use an environment
461 * variable here, we should really be writing out a configuration
462 * file and loading that on the next run. */
463 if (strcmp (mail_directory, default_path)) {
464 printf ("Note: Since you are not using the default path, you will want to set\n"
465 "the NOTMUCH_BASE environment variable to %s so that\n"
466 "future calls to notmuch commands will know where to find your mail.\n",
468 printf ("For example, if you are using bash for your shell, add:\n\n");
469 printf ("\texport NOTMUCH_BASE=%s\n\n", mail_directory);
470 printf ("to your ~/.bashrc file.\n\n");
475 notmuch = notmuch_database_create (mail_directory);
476 if (notmuch == NULL) {
477 fprintf (stderr, "Failed to create new notmuch database at %s\n",
479 ret = NOTMUCH_STATUS_FILE_ERROR;
483 printf ("OK. Let's take a look at the mail we can find in the directory\n");
484 printf ("%s ...\n", mail_directory);
487 count_files (mail_directory, &count);
489 printf ("Found %d total files. That's not much mail.\n\n", count);
491 printf ("Next, we'll inspect the messages and create a database of threads:\n");
493 add_files_state.ignore_read_only_directories = FALSE;
494 add_files_state.saw_read_only_directory = FALSE;
495 add_files_state.total_files = count;
496 add_files_state.processed_files = 0;
497 add_files_state.added_messages = 0;
498 gettimeofday (&add_files_state.tv_start, NULL);
500 ret = add_files (notmuch, mail_directory, &add_files_state);
502 gettimeofday (&tv_now, NULL);
503 elapsed = tv_elapsed (add_files_state.tv_start,
505 printf ("Processed %d %s in ", add_files_state.processed_files,
506 add_files_state.processed_files == 1 ?
507 "file" : "total files");
508 print_formatted_seconds (elapsed);
510 printf (" (%d files/sec.). \n",
511 (int) (add_files_state.processed_files / elapsed));
515 if (add_files_state.added_messages) {
516 printf ("Added %d %s to the database.\n\n",
517 add_files_state.added_messages,
518 add_files_state.added_messages == 1 ?
519 "message" : "unique messages");
522 printf ("When new mail is delivered to %s in the future,\n"
523 "run \"notmuch new\" to add it to the database.\n\n",
527 printf ("Note: At least one error was encountered: %s\n",
528 notmuch_status_to_string (ret));
533 free (mail_directory);
535 notmuch_database_close (notmuch);
541 new_command (unused (int argc), unused (char *argv[]))
543 notmuch_database_t *notmuch;
544 const char *mail_directory;
545 add_files_state_t add_files_state;
547 struct timeval tv_now;
550 notmuch = notmuch_database_open (NULL);
551 if (notmuch == NULL) {
556 mail_directory = notmuch_database_get_path (notmuch);
558 add_files_state.ignore_read_only_directories = TRUE;
559 add_files_state.saw_read_only_directory = FALSE;
560 add_files_state.total_files = 0;
561 add_files_state.processed_files = 0;
562 add_files_state.added_messages = 0;
563 gettimeofday (&add_files_state.tv_start, NULL);
565 ret = add_files (notmuch, mail_directory, &add_files_state);
567 gettimeofday (&tv_now, NULL);
568 elapsed = tv_elapsed (add_files_state.tv_start,
570 if (add_files_state.processed_files) {
571 printf ("Processed %d %s in ", add_files_state.processed_files,
572 add_files_state.processed_files == 1 ?
573 "file" : "total files");
574 print_formatted_seconds (elapsed);
576 printf (" (%d files/sec.). \n",
577 (int) (add_files_state.processed_files / elapsed));
582 if (add_files_state.added_messages) {
583 printf ("Added %d new %s to the database (not much, really).\n",
584 add_files_state.added_messages,
585 add_files_state.added_messages == 1 ?
586 "message" : "messages");
588 printf ("No new mail---and that's not much.\n");
591 if (elapsed > 1 && ! add_files_state.saw_read_only_directory) {
592 printf ("\nTip: If you have any sub-directories that are archives (that is,\n"
593 "they will never receive new mail), marking these directores as\n"
594 "read-only (chmod u-w /path/to/dir) will make \"notmuch new\"\n"
595 "much more efficient (it won't even look in those directories).\n");
599 printf ("\nNote: At least one error was encountered: %s\n",
600 notmuch_status_to_string (ret));
605 notmuch_database_close (notmuch);
611 search_command (int argc, char *argv[])
613 void *local = talloc_new (NULL);
614 notmuch_database_t *notmuch = NULL;
615 notmuch_query_t *query;
616 notmuch_thread_results_t *results;
617 notmuch_thread_t *thread;
618 notmuch_tags_t *tags;
621 notmuch_status_t ret = NOTMUCH_STATUS_SUCCESS;
623 notmuch = notmuch_database_open (NULL);
624 if (notmuch == NULL) {
629 /* XXX: Should add xtalloc wrappers here and use them. */
630 query_str = talloc_strdup (local, "");
632 for (i = 0; i < argc; i++) {
634 query_str = talloc_asprintf_append (query_str, " ");
636 query_str = talloc_asprintf_append (query_str, "%s", argv[i]);
639 query = notmuch_query_create (notmuch, query_str);
641 fprintf (stderr, "Out of memory\n");
646 for (results = notmuch_query_search_threads (query);
647 notmuch_thread_results_has_more (results);
648 notmuch_thread_results_advance (results))
652 thread = notmuch_thread_results_get (results);
655 notmuch_thread_get_thread_id (thread),
656 _notmuch_thread_get_subject (thread));
659 for (tags = notmuch_thread_get_tags (thread);
660 notmuch_tags_has_more (tags);
661 notmuch_tags_advance (tags))
665 printf ("%s", notmuch_tags_get (tags));
670 notmuch_thread_destroy (thread);
673 notmuch_query_destroy (query);
677 notmuch_database_close (notmuch);
684 show_command (unused (int argc), unused (char *argv[]))
686 fprintf (stderr, "Error: show is not implemented yet.\n");
691 dump_command (int argc, char *argv[])
694 notmuch_database_t *notmuch = NULL;
695 notmuch_query_t *query;
696 notmuch_message_results_t *results;
697 notmuch_message_t *message;
698 notmuch_tags_t *tags;
702 output = fopen (argv[0], "w");
703 if (output == NULL) {
704 fprintf (stderr, "Error opening %s for writing: %s\n",
705 argv[0], strerror (errno));
713 notmuch = notmuch_database_open (NULL);
714 if (notmuch == NULL) {
719 query = notmuch_query_create (notmuch, "");
721 fprintf (stderr, "Out of memory\n");
726 notmuch_query_set_sort (query, NOTMUCH_SORT_MESSAGE_ID);
728 for (results = notmuch_query_search_messages (query);
729 notmuch_message_results_has_more (results);
730 notmuch_message_results_advance (results))
733 message = notmuch_message_results_get (results);
736 "%s (", notmuch_message_get_message_id (message));
738 for (tags = notmuch_message_get_tags (message);
739 notmuch_tags_has_more (tags);
740 notmuch_tags_advance (tags))
743 fprintf (output, " ");
745 fprintf (output, "%s", notmuch_tags_get (tags));
750 fprintf (output, ")\n");
752 notmuch_message_destroy (message);
755 notmuch_query_destroy (query);
759 notmuch_database_close (notmuch);
760 if (output && output != stdout)
767 restore_command (int argc, char *argv[])
770 notmuch_database_t *notmuch = NULL;
779 input = fopen (argv[0], "r");
781 fprintf (stderr, "Error opening %s for reading: %s\n",
782 argv[0], strerror (errno));
787 printf ("No filename given. Reading dump from stdin.\n");
791 notmuch = notmuch_database_open (NULL);
792 if (notmuch == NULL) {
797 /* Dump output is one line per message. We match a sequence of
798 * non-space characters for the message-id, then one or more
799 * spaces, then a list of space-separated tags as a sequence of
800 * characters within literal '(' and ')'. */
802 "^([^ ]+) \\(([^)]*)\\)$",
805 while ((line_len = getline (&line, &line_size, input)) != -1) {
807 char *message_id, *tags, *tag, *next;
808 notmuch_message_t *message;
809 notmuch_status_t status;
811 chomp_newline (line);
813 rerr = xregexec (®ex, line, 3, match, 0);
814 if (rerr == REG_NOMATCH)
816 fprintf (stderr, "Warning: Ignoring invalid input line: %s\n",
821 message_id = xstrndup (line + match[1].rm_so,
822 match[1].rm_eo - match[1].rm_so);
823 tags = xstrndup (line + match[2].rm_so,
824 match[2].rm_eo - match[2].rm_so);
828 message = notmuch_database_find_message (notmuch, message_id);
829 if (message == NULL) {
830 fprintf (stderr, "Warning: Cannot apply tags to missing message: %s (",
836 tag = strsep (&next, " ");
840 status = notmuch_message_add_tag (message, tag);
843 "Error applying tag %s to message %s:\n",
845 fprintf (stderr, "%s\n",
846 notmuch_status_to_string (status));
849 fprintf (stderr, "%s%s",
850 tag == tags ? "" : " ", tag);
855 notmuch_message_destroy (message);
857 fprintf (stderr, ")\n");
869 notmuch_database_close (notmuch);
870 if (input && input != stdin)
876 command_t commands[] = {
877 { "setup", setup_command,
878 "Interactively setup notmuch for first use.\n\n"
879 "\t\tInvoking notmuch with no command argument will run setup if\n"
880 "\t\tthe setup command has not previously been completed." },
881 { "new", new_command,
882 "Find and import any new messages.\n\n"
883 "\t\tScans all sub-directories of the database, adding new files\n"
884 "\t\tthat are found. Note: \"notmuch new\" will skip any\n"
885 "\t\tread-only directories, so you can use that to mark\n"
886 "\t\tdirectories that will not receive any new mail."},
887 { "search", search_command,
888 "<search-term> [...]\n\n"
889 "\t\tSearch for threads matching the given search terms.\n"
890 "\t\tOnce we actually implement search we'll document the\n"
891 "\t\tsyntax here." },
892 { "show", show_command,
894 "\t\tShow the thread with the given thread ID (see 'search')." },
895 { "dump", dump_command,
897 "\t\tCreate a plain-text dump of the tags for each message\n"
898 "\t\twriting to the given filename, if any, or to stdout.\n"
899 "\t\tThese tags are the only data in the notmuch database\n"
900 "\t\tthat can't be recreated from the messages themselves.\n"
901 "\t\tThe output of notmuch dump is therefore the only\n"
902 "\t\tcritical thing to backup (and much more friendly to\n"
903 "\t\tincremental backup than the native database files." },
904 { "restore", restore_command,
906 "\t\tRestore the tags from the given dump file (see 'dump')." }
915 fprintf (stderr, "Usage: notmuch <command> [args...]\n");
916 fprintf (stderr, "\n");
917 fprintf (stderr, "Where <command> and [args...] are as follows:\n");
918 fprintf (stderr, "\n");
920 for (i = 0; i < ARRAY_SIZE (commands); i++) {
921 command = &commands[i];
923 fprintf (stderr, "\t%s\t%s\n\n", command->name, command->usage);
928 main (int argc, char *argv[])
934 return setup_command (0, NULL);
936 for (i = 0; i < ARRAY_SIZE (commands); i++) {
937 command = &commands[i];
939 if (strcmp (argv[1], command->name) == 0)
940 return (command->function) (argc - 2, &argv[2]);
943 /* Don't complain about "help" being an unknown command when we're
944 about to provide exactly what's wanted anyway. */
945 if (strcmp (argv[1], "help") == 0 ||
946 strcmp (argv[1], "--help") == 0)
948 fprintf (stderr, "The notmuch mail system.\n\n");
952 fprintf (stderr, "Error: Unknown command '%s'\n\n", argv[1]);