1 /* notmuch - Not much of an email program, (just index and search)
3 * Copyright © 2009 Carl Worth
5 * This program is free software: you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation, either version 3 of the License, or
8 * (at your option) any later version.
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
15 * You should have received a copy of the GNU General Public License
16 * along with this program. If not, see http://www.gnu.org/licenses/ .
18 * Author: Carl Worth <cworth@cworth.org>
22 #define _GNU_SOURCE /* for getline */
28 /* This is separate from notmuch-private.h because we're trying to
29 * keep notmuch.c from looking into any internals, (which helps us
30 * develop notmuch.h into a plausible library interface).
44 #define unused(x) x __attribute__ ((unused))
46 /* There's no point in continuing when we've detected that we've done
47 * something wrong internally (as opposed to the user passing in a
50 * Note that __location__ comes from talloc.h.
52 #define INTERNAL_ERROR(format, ...) \
55 "Internal error: " format " (%s)\n", \
56 ##__VA_ARGS__, __location__); \
60 #define ARRAY_SIZE(arr) (sizeof (arr) / sizeof (arr[0]))
62 typedef int (*command_function_t) (int argc, char *argv[]);
64 typedef struct command {
66 command_function_t function;
71 int ignore_read_only_directories;
72 int saw_read_only_directory;
77 struct timeval tv_start;
81 chomp_newline (char *str)
83 if (str && str[strlen(str)-1] == '\n')
84 str[strlen(str)-1] = '\0';
87 /* Compute the number of seconds elapsed from start to end. */
89 tv_elapsed (struct timeval start, struct timeval end)
91 return ((end.tv_sec - start.tv_sec) +
92 (end.tv_usec - start.tv_usec) / 1e6);
96 print_formatted_seconds (double seconds)
102 printf ("almost no time");
106 if (seconds > 3600) {
107 hours = (int) seconds / 3600;
108 printf ("%dh ", hours);
109 seconds -= hours * 3600;
113 minutes = (int) seconds / 60;
114 printf ("%dm ", minutes);
115 seconds -= minutes * 60;
118 printf ("%ds", (int) seconds);
122 add_files_print_progress (add_files_state_t *state)
124 struct timeval tv_now;
125 double elapsed_overall, rate_overall;
127 gettimeofday (&tv_now, NULL);
129 elapsed_overall = tv_elapsed (state->tv_start, tv_now);
130 rate_overall = (state->processed_files) / elapsed_overall;
132 printf ("Processed %d", state->processed_files);
134 if (state->total_files) {
135 printf (" of %d files (", state->total_files);
136 print_formatted_seconds ((state->total_files - state->processed_files) /
138 printf (" remaining). \r");
140 printf (" files (%d files/sec.) \r", (int) rate_overall);
146 /* Examine 'path' recursively as follows:
148 * o Ask the filesystem for the mtime of 'path' (path_mtime)
150 * o Ask the database for its timestamp of 'path' (path_dbtime)
152 * o If 'path_mtime' > 'path_dbtime'
154 * o For each regular file in 'path' with mtime newer than the
155 * 'path_dbtime' call add_message to add the file to the
158 * o For each sub-directory of path, recursively call into this
161 * o Tell the database to update its time of 'path' to 'path_mtime'
163 * The 'struct stat *st' must point to a structure that has already
164 * been initialized for 'path' by calling stat().
166 static notmuch_status_t
167 add_files_recursive (notmuch_database_t *notmuch,
170 add_files_state_t *state)
173 struct dirent *e, *entry = NULL;
177 time_t path_mtime, path_dbtime;
178 notmuch_status_t status, ret = NOTMUCH_STATUS_SUCCESS;
180 /* If we're told to, we bail out on encountering a read-only
181 * directory, (with this being a clear clue from the user to
182 * Notmuch that new mail won't be arriving there and we need not
184 if (state->ignore_read_only_directories &&
185 (st->st_mode & S_IWUSR) == 0)
187 state->saw_read_only_directory = TRUE;
191 path_mtime = st->st_mtime;
193 path_dbtime = notmuch_database_get_timestamp (notmuch, path);
195 dir = opendir (path);
197 fprintf (stderr, "Error opening directory %s: %s\n",
198 path, strerror (errno));
199 ret = NOTMUCH_STATUS_FILE_ERROR;
203 entry_length = offsetof (struct dirent, d_name) +
204 pathconf (path, _PC_NAME_MAX) + 1;
205 entry = malloc (entry_length);
208 err = readdir_r (dir, entry, &e);
210 fprintf (stderr, "Error reading directory: %s\n",
212 ret = NOTMUCH_STATUS_FILE_ERROR;
219 /* If this directory hasn't been modified since the last
220 * add_files, then we only need to look further for
221 * sub-directories. */
222 if (path_mtime <= path_dbtime && entry->d_type != DT_DIR)
225 /* Ignore special directories to avoid infinite recursion.
226 * Also ignore the .notmuch directory.
228 /* XXX: Eventually we'll want more sophistication to let the
229 * user specify files to be ignored. */
230 if (strcmp (entry->d_name, ".") == 0 ||
231 strcmp (entry->d_name, "..") == 0 ||
232 strcmp (entry->d_name, ".notmuch") ==0)
237 next = talloc_asprintf (notmuch, "%s/%s", path, entry->d_name);
239 if (stat (next, st)) {
240 fprintf (stderr, "Error reading %s: %s\n",
241 next, strerror (errno));
242 ret = NOTMUCH_STATUS_FILE_ERROR;
246 if (S_ISREG (st->st_mode)) {
247 /* If the file hasn't been modified since the last
248 * add_files, then we need not look at it. */
249 if (st->st_mtime > path_dbtime) {
250 state->processed_files++;
252 status = notmuch_database_add_message (notmuch, next);
255 case NOTMUCH_STATUS_SUCCESS:
256 state->added_messages++;
258 /* Non-fatal issues (go on to next file) */
259 case NOTMUCH_STATUS_DUPLICATE_MESSAGE_ID:
260 /* Stay silent on this one. */
262 case NOTMUCH_STATUS_FILE_NOT_EMAIL:
263 fprintf (stderr, "Note: Ignoring non-mail file: %s\n",
266 /* Fatal issues. Don't process anymore. */
267 case NOTMUCH_STATUS_XAPIAN_EXCEPTION:
268 case NOTMUCH_STATUS_OUT_OF_MEMORY:
269 fprintf (stderr, "Error: %s. Halting processing.\n",
270 notmuch_status_to_string (status));
274 case NOTMUCH_STATUS_FILE_ERROR:
275 case NOTMUCH_STATUS_NULL_POINTER:
276 case NOTMUCH_STATUS_TAG_TOO_LONG:
277 case NOTMUCH_STATUS_LAST_STATUS:
278 INTERNAL_ERROR ("add_message returned unexpected value: %d", status);
281 if (state->processed_files % 1000 == 0)
282 add_files_print_progress (state);
284 } else if (S_ISDIR (st->st_mode)) {
285 status = add_files_recursive (notmuch, next, st, state);
286 if (status && ret == NOTMUCH_STATUS_SUCCESS)
294 status = notmuch_database_set_timestamp (notmuch, path, path_mtime);
295 if (status && ret == NOTMUCH_STATUS_SUCCESS)
309 /* This is the top-level entry point for add_files. It does a couple
310 * of error checks, and then calls into the recursive function,
311 * (avoiding the repeating of these error checks at every
312 * level---which would be useless becaues we already do a stat() at
313 * the level above). */
314 static notmuch_status_t
315 add_files (notmuch_database_t *notmuch,
317 add_files_state_t *state)
321 if (stat (path, &st)) {
322 fprintf (stderr, "Error reading directory %s: %s\n",
323 path, strerror (errno));
324 return NOTMUCH_STATUS_FILE_ERROR;
327 if (! S_ISDIR (st.st_mode)) {
328 fprintf (stderr, "Error: %s is not a directory.\n", path);
329 return NOTMUCH_STATUS_FILE_ERROR;
332 return add_files_recursive (notmuch, path, &st, state);
335 /* Recursively count all regular files in path and all sub-direcotries
336 * of path. The result is added to *count (which should be
337 * initialized to zero by the top-level caller before calling
340 count_files (const char *path, int *count)
343 struct dirent *e, *entry = NULL;
349 dir = opendir (path);
352 fprintf (stderr, "Warning: failed to open directory %s: %s\n",
353 path, strerror (errno));
357 entry_length = offsetof (struct dirent, d_name) +
358 pathconf (path, _PC_NAME_MAX) + 1;
359 entry = malloc (entry_length);
362 err = readdir_r (dir, entry, &e);
364 fprintf (stderr, "Error reading directory: %s\n",
373 /* Ignore special directories to avoid infinite recursion.
374 * Also ignore the .notmuch directory.
376 /* XXX: Eventually we'll want more sophistication to let the
377 * user specify files to be ignored. */
378 if (strcmp (entry->d_name, ".") == 0 ||
379 strcmp (entry->d_name, "..") == 0 ||
380 strcmp (entry->d_name, ".notmuch") == 0)
385 if (asprintf (&next, "%s/%s", path, entry->d_name) == -1) {
387 fprintf (stderr, "Error descending from %s to %s: Out of memory\n",
388 path, entry->d_name);
394 if (S_ISREG (st.st_mode)) {
396 if (*count % 1000 == 0) {
397 printf ("Found %d files so far.\r", *count);
400 } else if (S_ISDIR (st.st_mode)) {
401 count_files (next, count);
415 setup_command (unused (int argc), unused (char *argv[]))
417 notmuch_database_t *notmuch = NULL;
418 char *default_path, *mail_directory = NULL;
421 add_files_state_t add_files_state;
423 struct timeval tv_now;
424 notmuch_status_t ret = NOTMUCH_STATUS_SUCCESS;
426 printf ("Welcome to notmuch!\n\n");
428 printf ("The goal of notmuch is to help you manage and search your collection of\n"
429 "email, and to efficiently keep up with the flow of email as it comes in.\n\n");
431 printf ("Notmuch needs to know the top-level directory of your email archive,\n"
432 "(where you already have mail stored and where messages will be delivered\n"
433 "in the future). This directory can contain any number of sub-directories\n"
434 "and primarily just files with indvidual email messages (eg. maildir or mh\n"
435 "archives are perfect). If there are other, non-email files (such as\n"
436 "indexes maintained by other email programs) then notmuch will do its\n"
437 "best to detect those and ignore them.\n\n");
439 printf ("Mail storage that uses mbox format, (where one mbox file contains many\n"
440 "messages), will not work with notmuch. If that's how your mail is currently\n"
441 "stored, we recommend you first convert it to maildir format with a utility\n"
442 "such as mb2md. In that case, press Control-C now and run notmuch again\n"
443 "once the conversion is complete.\n\n");
446 default_path = notmuch_database_default_path ();
447 printf ("Top-level mail directory [%s]: ", default_path);
450 getline (&mail_directory, &line_size, stdin);
451 chomp_newline (mail_directory);
455 if (mail_directory == NULL || strlen (mail_directory) == 0) {
457 free (mail_directory);
458 mail_directory = default_path;
460 /* XXX: Instead of telling the user to use an environment
461 * variable here, we should really be writing out a configuration
462 * file and loading that on the next run. */
463 if (strcmp (mail_directory, default_path)) {
464 printf ("Note: Since you are not using the default path, you will want to set\n"
465 "the NOTMUCH_BASE environment variable to %s so that\n"
466 "future calls to notmuch commands will know where to find your mail.\n",
468 printf ("For example, if you are using bash for your shell, add:\n\n");
469 printf ("\texport NOTMUCH_BASE=%s\n\n", mail_directory);
470 printf ("to your ~/.bashrc file.\n\n");
475 notmuch = notmuch_database_create (mail_directory);
476 if (notmuch == NULL) {
477 fprintf (stderr, "Failed to create new notmuch database at %s\n",
479 ret = NOTMUCH_STATUS_FILE_ERROR;
483 printf ("OK. Let's take a look at the mail we can find in the directory\n");
484 printf ("%s ...\n", mail_directory);
487 count_files (mail_directory, &count);
489 printf ("Found %d total files. That's not much mail.\n\n", count);
491 printf ("Next, we'll inspect the messages and create a database of threads:\n");
493 add_files_state.ignore_read_only_directories = FALSE;
494 add_files_state.saw_read_only_directory = FALSE;
495 add_files_state.total_files = count;
496 add_files_state.processed_files = 0;
497 add_files_state.added_messages = 0;
498 gettimeofday (&add_files_state.tv_start, NULL);
500 ret = add_files (notmuch, mail_directory, &add_files_state);
502 gettimeofday (&tv_now, NULL);
503 elapsed = tv_elapsed (add_files_state.tv_start,
505 printf ("Processed %d %s in ", add_files_state.processed_files,
506 add_files_state.processed_files == 1 ?
507 "file" : "total files");
508 print_formatted_seconds (elapsed);
510 printf (" (%d files/sec.). \n",
511 (int) (add_files_state.processed_files / elapsed));
515 if (add_files_state.added_messages) {
516 printf ("Added %d %s to the database.\n\n",
517 add_files_state.added_messages,
518 add_files_state.added_messages == 1 ?
519 "message" : "unique messages");
522 printf ("When new mail is delivered to %s in the future,\n"
523 "run \"notmuch new\" to add it to the database.\n\n",
527 printf ("Note: At least one error was encountered: %s\n",
528 notmuch_status_to_string (ret));
533 free (mail_directory);
535 notmuch_database_close (notmuch);
541 new_command (unused (int argc), unused (char *argv[]))
543 notmuch_database_t *notmuch;
544 const char *mail_directory;
545 add_files_state_t add_files_state;
547 struct timeval tv_now;
550 notmuch = notmuch_database_open (NULL);
551 if (notmuch == NULL) {
556 mail_directory = notmuch_database_get_path (notmuch);
558 add_files_state.ignore_read_only_directories = TRUE;
559 add_files_state.saw_read_only_directory = FALSE;
560 add_files_state.total_files = 0;
561 add_files_state.processed_files = 0;
562 add_files_state.added_messages = 0;
563 gettimeofday (&add_files_state.tv_start, NULL);
565 ret = add_files (notmuch, mail_directory, &add_files_state);
567 gettimeofday (&tv_now, NULL);
568 elapsed = tv_elapsed (add_files_state.tv_start,
570 if (add_files_state.processed_files) {
571 printf ("Processed %d %s in ", add_files_state.processed_files,
572 add_files_state.processed_files == 1 ?
573 "file" : "total files");
574 print_formatted_seconds (elapsed);
576 printf (" (%d files/sec.). \n",
577 (int) (add_files_state.processed_files / elapsed));
582 if (add_files_state.added_messages) {
583 printf ("Added %d new %s to the database (not much, really).\n",
584 add_files_state.added_messages,
585 add_files_state.added_messages == 1 ?
586 "message" : "messages");
588 printf ("No new mail---and that's not much.\n");
591 if (elapsed > 1 && ! add_files_state.saw_read_only_directory) {
592 printf ("\nTip: If you have any sub-directories that are archives (that is,\n"
593 "they will never receive new mail), marking these directores as\n"
594 "read-only (chmod u-w /path/to/dir) will make \"notmuch new\"\n"
595 "much more efficient (it won't even look in those directories).\n");
599 printf ("\nNote: At least one error was encountered: %s\n",
600 notmuch_status_to_string (ret));
605 notmuch_database_close (notmuch);
611 search_command (int argc, char *argv[])
613 void *local = talloc_new (NULL);
614 notmuch_database_t *notmuch = NULL;
615 notmuch_query_t *query;
616 notmuch_thread_results_t *results;
617 notmuch_thread_t *thread;
618 notmuch_tags_t *tags;
621 notmuch_status_t ret = NOTMUCH_STATUS_SUCCESS;
623 notmuch = notmuch_database_open (NULL);
624 if (notmuch == NULL) {
629 /* XXX: Should add xtalloc wrappers here and use them. */
630 query_str = talloc_strdup (local, "");
632 for (i = 0; i < argc; i++) {
634 query_str = talloc_asprintf_append (query_str, " ");
636 query_str = talloc_asprintf_append (query_str, "%s", argv[i]);
639 query = notmuch_query_create (notmuch, query_str);
641 fprintf (stderr, "Out of memory\n");
646 for (results = notmuch_query_search_threads (query);
647 notmuch_thread_results_has_more (results);
648 notmuch_thread_results_advance (results))
652 thread = notmuch_thread_results_get (results);
654 printf ("%s (", notmuch_thread_get_thread_id (thread));
656 for (tags = notmuch_thread_get_tags (thread);
657 notmuch_tags_has_more (tags);
658 notmuch_tags_advance (tags))
662 printf ("%s", notmuch_tags_get (tags));
668 notmuch_thread_destroy (thread);
671 notmuch_query_destroy (query);
675 notmuch_database_close (notmuch);
682 show_command (unused (int argc), unused (char *argv[]))
684 fprintf (stderr, "Error: show is not implemented yet.\n");
689 dump_command (int argc, char *argv[])
692 notmuch_database_t *notmuch = NULL;
693 notmuch_query_t *query;
694 notmuch_message_results_t *results;
695 notmuch_message_t *message;
696 notmuch_tags_t *tags;
700 output = fopen (argv[0], "w");
701 if (output == NULL) {
702 fprintf (stderr, "Error opening %s for writing: %s\n",
703 argv[0], strerror (errno));
711 notmuch = notmuch_database_open (NULL);
712 if (notmuch == NULL) {
717 query = notmuch_query_create (notmuch, "");
719 fprintf (stderr, "Out of memory\n");
724 notmuch_query_set_sort (query, NOTMUCH_SORT_MESSAGE_ID);
726 for (results = notmuch_query_search_messages (query);
727 notmuch_message_results_has_more (results);
728 notmuch_message_results_advance (results))
731 message = notmuch_message_results_get (results);
734 "%s (", notmuch_message_get_message_id (message));
736 for (tags = notmuch_message_get_tags (message);
737 notmuch_tags_has_more (tags);
738 notmuch_tags_advance (tags))
741 fprintf (output, " ");
743 fprintf (output, "%s", notmuch_tags_get (tags));
748 fprintf (output, ")\n");
750 notmuch_message_destroy (message);
753 notmuch_query_destroy (query);
757 notmuch_database_close (notmuch);
758 if (output && output != stdout)
765 restore_command (int argc, char *argv[])
768 notmuch_database_t *notmuch = NULL;
777 input = fopen (argv[0], "r");
779 fprintf (stderr, "Error opening %s for reading: %s\n",
780 argv[0], strerror (errno));
785 printf ("No filename given. Reading dump from stdin.\n");
789 notmuch = notmuch_database_open (NULL);
790 if (notmuch == NULL) {
795 /* Dump output is one line per message. We match a sequence of
796 * non-space characters for the message-id, then one or more
797 * spaces, then a list of space-separated tags as a sequence of
798 * characters within literal '(' and ')'. */
800 "^([^ ]+) \\(([^)]*)\\)$",
803 while ((line_len = getline (&line, &line_size, input)) != -1) {
805 char *message_id, *tags, *tag, *next;
806 notmuch_message_t *message;
807 notmuch_status_t status;
809 chomp_newline (line);
811 rerr = xregexec (®ex, line, 3, match, 0);
812 if (rerr == REG_NOMATCH)
814 fprintf (stderr, "Warning: Ignoring invalid input line: %s\n",
819 message_id = xstrndup (line + match[1].rm_so,
820 match[1].rm_eo - match[1].rm_so);
821 tags = xstrndup (line + match[2].rm_so,
822 match[2].rm_eo - match[2].rm_so);
826 message = notmuch_database_find_message (notmuch, message_id);
827 if (message == NULL) {
828 fprintf (stderr, "Warning: Cannot apply tags to missing message: %s (",
834 tag = strsep (&next, " ");
838 status = notmuch_message_add_tag (message, tag);
841 "Error applying tag %s to message %s:\n",
843 fprintf (stderr, "%s\n",
844 notmuch_status_to_string (status));
847 fprintf (stderr, "%s%s",
848 tag == tags ? "" : " ", tag);
853 notmuch_message_destroy (message);
855 fprintf (stderr, ")\n");
867 notmuch_database_close (notmuch);
868 if (input && input != stdin)
874 command_t commands[] = {
875 { "setup", setup_command,
876 "Interactively setup notmuch for first use.\n\n"
877 "\t\tInvoking notmuch with no command argument will run setup if\n"
878 "\t\tthe setup command has not previously been completed." },
879 { "new", new_command,
880 "Find and import any new messages.\n\n"
881 "\t\tScans all sub-directories of the database, adding new files\n"
882 "\t\tthat are found. Note: \"notmuch new\" will skip any\n"
883 "\t\tread-only directories, so you can use that to mark\n"
884 "\t\tdirectories that will not receive any new mail."},
885 { "search", search_command,
886 "<search-term> [...]\n\n"
887 "\t\tSearch for threads matching the given search terms.\n"
888 "\t\tOnce we actually implement search we'll document the\n"
889 "\t\tsyntax here." },
890 { "show", show_command,
892 "\t\tShow the thread with the given thread ID (see 'search')." },
893 { "dump", dump_command,
895 "\t\tCreate a plain-text dump of the tags for each message\n"
896 "\t\twriting to the given filename, if any, or to stdout.\n"
897 "\t\tThese tags are the only data in the notmuch database\n"
898 "\t\tthat can't be recreated from the messages themselves.\n"
899 "\t\tThe output of notmuch dump is therefore the only\n"
900 "\t\tcritical thing to backup (and much more friendly to\n"
901 "\t\tincremental backup than the native database files." },
902 { "restore", restore_command,
904 "\t\tRestore the tags from the given dump file (see 'dump')." }
913 fprintf (stderr, "Usage: notmuch <command> [args...]\n");
914 fprintf (stderr, "\n");
915 fprintf (stderr, "Where <command> and [args...] are as follows:\n");
916 fprintf (stderr, "\n");
918 for (i = 0; i < ARRAY_SIZE (commands); i++) {
919 command = &commands[i];
921 fprintf (stderr, "\t%s\t%s\n\n", command->name, command->usage);
926 main (int argc, char *argv[])
932 return setup_command (0, NULL);
934 for (i = 0; i < ARRAY_SIZE (commands); i++) {
935 command = &commands[i];
937 if (strcmp (argv[1], command->name) == 0)
938 return (command->function) (argc - 2, &argv[2]);
941 /* Don't complain about "help" being an unknown command when we're
942 about to provide exactly what's wanted anyway. */
943 if (strcmp (argv[1], "help") == 0 ||
944 strcmp (argv[1], "--help") == 0)
946 fprintf (stderr, "The notmuch mail system.\n\n");
950 fprintf (stderr, "Error: Unknown command '%s'\n\n", argv[1]);