1 /* notmuch - Not much of an email program, (just index and search)
3 * Copyright © 2009 Carl Worth
5 * This program is free software: you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation, either version 3 of the License, or
8 * (at your option) any later version.
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
15 * You should have received a copy of the GNU General Public License
16 * along with this program. If not, see http://www.gnu.org/licenses/ .
18 * Author: Carl Worth <cworth@cworth.org>
22 #define _GNU_SOURCE /* for getline */
28 /* This is separate from notmuch-private.h because we're trying to
29 * keep notmuch.c from looking into any internals, (which helps us
30 * develop notmuch.h into a plausible library interface).
44 #define unused(x) x __attribute__ ((unused))
46 /* There's no point in continuing when we've detected that we've done
47 * something wrong internally (as opposed to the user passing in a
50 * Note that __location__ comes from talloc.h.
52 #define INTERNAL_ERROR(format, ...) \
55 "Internal error: " format " (%s)\n", \
56 ##__VA_ARGS__, __location__); \
60 #define ARRAY_SIZE(arr) (sizeof (arr) / sizeof (arr[0]))
62 typedef int (*command_function_t) (int argc, char *argv[]);
64 typedef struct command {
66 command_function_t function;
70 typedef void (*add_files_callback_t) (notmuch_message_t *message);
73 int ignore_read_only_directories;
74 int saw_read_only_directory;
79 struct timeval tv_start;
81 add_files_callback_t callback;
85 chomp_newline (char *str)
87 if (str && str[strlen(str)-1] == '\n')
88 str[strlen(str)-1] = '\0';
91 /* Compute the number of seconds elapsed from start to end. */
93 tv_elapsed (struct timeval start, struct timeval end)
95 return ((end.tv_sec - start.tv_sec) +
96 (end.tv_usec - start.tv_usec) / 1e6);
100 print_formatted_seconds (double seconds)
106 printf ("almost no time");
110 if (seconds > 3600) {
111 hours = (int) seconds / 3600;
112 printf ("%dh ", hours);
113 seconds -= hours * 3600;
117 minutes = (int) seconds / 60;
118 printf ("%dm ", minutes);
119 seconds -= minutes * 60;
122 printf ("%ds", (int) seconds);
126 add_files_print_progress (add_files_state_t *state)
128 struct timeval tv_now;
129 double elapsed_overall, rate_overall;
131 gettimeofday (&tv_now, NULL);
133 elapsed_overall = tv_elapsed (state->tv_start, tv_now);
134 rate_overall = (state->processed_files) / elapsed_overall;
136 printf ("Processed %d", state->processed_files);
138 if (state->total_files) {
139 printf (" of %d files (", state->total_files);
140 print_formatted_seconds ((state->total_files - state->processed_files) /
142 printf (" remaining). \r");
144 printf (" files (%d files/sec.) \r", (int) rate_overall);
150 /* Examine 'path' recursively as follows:
152 * o Ask the filesystem for the mtime of 'path' (path_mtime)
154 * o Ask the database for its timestamp of 'path' (path_dbtime)
156 * o If 'path_mtime' > 'path_dbtime'
158 * o For each regular file in 'path' with mtime newer than the
159 * 'path_dbtime' call add_message to add the file to the
162 * o For each sub-directory of path, recursively call into this
165 * o Tell the database to update its time of 'path' to 'path_mtime'
167 * The 'struct stat *st' must point to a structure that has already
168 * been initialized for 'path' by calling stat().
170 static notmuch_status_t
171 add_files_recursive (notmuch_database_t *notmuch,
174 add_files_state_t *state)
177 struct dirent *e, *entry = NULL;
181 time_t path_mtime, path_dbtime;
182 notmuch_status_t status, ret = NOTMUCH_STATUS_SUCCESS;
183 notmuch_message_t *message, **closure;
185 /* If we're told to, we bail out on encountering a read-only
186 * directory, (with this being a clear clue from the user to
187 * Notmuch that new mail won't be arriving there and we need not
189 if (state->ignore_read_only_directories &&
190 (st->st_mode & S_IWUSR) == 0)
192 state->saw_read_only_directory = TRUE;
196 path_mtime = st->st_mtime;
198 path_dbtime = notmuch_database_get_timestamp (notmuch, path);
200 dir = opendir (path);
202 fprintf (stderr, "Error opening directory %s: %s\n",
203 path, strerror (errno));
204 ret = NOTMUCH_STATUS_FILE_ERROR;
208 entry_length = offsetof (struct dirent, d_name) +
209 pathconf (path, _PC_NAME_MAX) + 1;
210 entry = malloc (entry_length);
213 err = readdir_r (dir, entry, &e);
215 fprintf (stderr, "Error reading directory: %s\n",
217 ret = NOTMUCH_STATUS_FILE_ERROR;
224 /* If this directory hasn't been modified since the last
225 * add_files, then we only need to look further for
226 * sub-directories. */
227 if (path_mtime <= path_dbtime && entry->d_type != DT_DIR)
230 /* Ignore special directories to avoid infinite recursion.
231 * Also ignore the .notmuch directory.
233 /* XXX: Eventually we'll want more sophistication to let the
234 * user specify files to be ignored. */
235 if (strcmp (entry->d_name, ".") == 0 ||
236 strcmp (entry->d_name, "..") == 0 ||
237 strcmp (entry->d_name, ".notmuch") ==0)
242 next = talloc_asprintf (notmuch, "%s/%s", path, entry->d_name);
244 if (stat (next, st)) {
245 fprintf (stderr, "Error reading %s: %s\n",
246 next, strerror (errno));
247 ret = NOTMUCH_STATUS_FILE_ERROR;
251 if (S_ISREG (st->st_mode)) {
252 /* If the file hasn't been modified since the last
253 * add_files, then we need not look at it. */
254 if (st->st_mtime > path_dbtime) {
255 state->processed_files++;
262 status = notmuch_database_add_message (notmuch, next, closure);
265 case NOTMUCH_STATUS_SUCCESS:
266 state->added_messages++;
268 (state->callback) (message);
269 notmuch_message_destroy (message);
271 /* Non-fatal issues (go on to next file) */
272 case NOTMUCH_STATUS_DUPLICATE_MESSAGE_ID:
273 /* Stay silent on this one. */
275 case NOTMUCH_STATUS_FILE_NOT_EMAIL:
276 fprintf (stderr, "Note: Ignoring non-mail file: %s\n",
279 /* Fatal issues. Don't process anymore. */
280 case NOTMUCH_STATUS_XAPIAN_EXCEPTION:
281 case NOTMUCH_STATUS_OUT_OF_MEMORY:
282 fprintf (stderr, "Error: %s. Halting processing.\n",
283 notmuch_status_to_string (status));
287 case NOTMUCH_STATUS_FILE_ERROR:
288 case NOTMUCH_STATUS_NULL_POINTER:
289 case NOTMUCH_STATUS_TAG_TOO_LONG:
290 case NOTMUCH_STATUS_LAST_STATUS:
291 INTERNAL_ERROR ("add_message returned unexpected value: %d", status);
294 if (state->processed_files % 1000 == 0)
295 add_files_print_progress (state);
297 } else if (S_ISDIR (st->st_mode)) {
298 status = add_files_recursive (notmuch, next, st, state);
299 if (status && ret == NOTMUCH_STATUS_SUCCESS)
307 status = notmuch_database_set_timestamp (notmuch, path, path_mtime);
308 if (status && ret == NOTMUCH_STATUS_SUCCESS)
322 /* This is the top-level entry point for add_files. It does a couple
323 * of error checks, and then calls into the recursive function,
324 * (avoiding the repeating of these error checks at every
325 * level---which would be useless becaues we already do a stat() at
326 * the level above). */
327 static notmuch_status_t
328 add_files (notmuch_database_t *notmuch,
330 add_files_state_t *state)
334 if (stat (path, &st)) {
335 fprintf (stderr, "Error reading directory %s: %s\n",
336 path, strerror (errno));
337 return NOTMUCH_STATUS_FILE_ERROR;
340 if (! S_ISDIR (st.st_mode)) {
341 fprintf (stderr, "Error: %s is not a directory.\n", path);
342 return NOTMUCH_STATUS_FILE_ERROR;
345 return add_files_recursive (notmuch, path, &st, state);
348 /* Recursively count all regular files in path and all sub-direcotries
349 * of path. The result is added to *count (which should be
350 * initialized to zero by the top-level caller before calling
353 count_files (const char *path, int *count)
356 struct dirent *e, *entry = NULL;
362 dir = opendir (path);
365 fprintf (stderr, "Warning: failed to open directory %s: %s\n",
366 path, strerror (errno));
370 entry_length = offsetof (struct dirent, d_name) +
371 pathconf (path, _PC_NAME_MAX) + 1;
372 entry = malloc (entry_length);
375 err = readdir_r (dir, entry, &e);
377 fprintf (stderr, "Error reading directory: %s\n",
386 /* Ignore special directories to avoid infinite recursion.
387 * Also ignore the .notmuch directory.
389 /* XXX: Eventually we'll want more sophistication to let the
390 * user specify files to be ignored. */
391 if (strcmp (entry->d_name, ".") == 0 ||
392 strcmp (entry->d_name, "..") == 0 ||
393 strcmp (entry->d_name, ".notmuch") == 0)
398 if (asprintf (&next, "%s/%s", path, entry->d_name) == -1) {
400 fprintf (stderr, "Error descending from %s to %s: Out of memory\n",
401 path, entry->d_name);
407 if (S_ISREG (st.st_mode)) {
409 if (*count % 1000 == 0) {
410 printf ("Found %d files so far.\r", *count);
413 } else if (S_ISDIR (st.st_mode)) {
414 count_files (next, count);
428 setup_command (unused (int argc), unused (char *argv[]))
430 notmuch_database_t *notmuch = NULL;
431 char *default_path, *mail_directory = NULL;
434 add_files_state_t add_files_state;
436 struct timeval tv_now;
437 notmuch_status_t ret = NOTMUCH_STATUS_SUCCESS;
439 printf ("Welcome to notmuch!\n\n");
441 printf ("The goal of notmuch is to help you manage and search your collection of\n"
442 "email, and to efficiently keep up with the flow of email as it comes in.\n\n");
444 printf ("Notmuch needs to know the top-level directory of your email archive,\n"
445 "(where you already have mail stored and where messages will be delivered\n"
446 "in the future). This directory can contain any number of sub-directories\n"
447 "and primarily just files with indvidual email messages (eg. maildir or mh\n"
448 "archives are perfect). If there are other, non-email files (such as\n"
449 "indexes maintained by other email programs) then notmuch will do its\n"
450 "best to detect those and ignore them.\n\n");
452 printf ("Mail storage that uses mbox format, (where one mbox file contains many\n"
453 "messages), will not work with notmuch. If that's how your mail is currently\n"
454 "stored, we recommend you first convert it to maildir format with a utility\n"
455 "such as mb2md. In that case, press Control-C now and run notmuch again\n"
456 "once the conversion is complete.\n\n");
459 default_path = notmuch_database_default_path ();
460 printf ("Top-level mail directory [%s]: ", default_path);
463 getline (&mail_directory, &line_size, stdin);
464 chomp_newline (mail_directory);
468 if (mail_directory == NULL || strlen (mail_directory) == 0) {
470 free (mail_directory);
471 mail_directory = default_path;
473 /* XXX: Instead of telling the user to use an environment
474 * variable here, we should really be writing out a configuration
475 * file and loading that on the next run. */
476 if (strcmp (mail_directory, default_path)) {
477 printf ("Note: Since you are not using the default path, you will want to set\n"
478 "the NOTMUCH_BASE environment variable to %s so that\n"
479 "future calls to notmuch commands will know where to find your mail.\n",
481 printf ("For example, if you are using bash for your shell, add:\n\n");
482 printf ("\texport NOTMUCH_BASE=%s\n\n", mail_directory);
483 printf ("to your ~/.bashrc file.\n\n");
488 notmuch = notmuch_database_create (mail_directory);
489 if (notmuch == NULL) {
490 fprintf (stderr, "Failed to create new notmuch database at %s\n",
492 ret = NOTMUCH_STATUS_FILE_ERROR;
496 printf ("OK. Let's take a look at the mail we can find in the directory\n");
497 printf ("%s ...\n", mail_directory);
500 count_files (mail_directory, &count);
502 printf ("Found %d total files. That's not much mail.\n\n", count);
504 printf ("Next, we'll inspect the messages and create a database of threads:\n");
506 add_files_state.ignore_read_only_directories = FALSE;
507 add_files_state.saw_read_only_directory = FALSE;
508 add_files_state.total_files = count;
509 add_files_state.processed_files = 0;
510 add_files_state.added_messages = 0;
511 add_files_state.callback = NULL;
512 gettimeofday (&add_files_state.tv_start, NULL);
514 ret = add_files (notmuch, mail_directory, &add_files_state);
516 gettimeofday (&tv_now, NULL);
517 elapsed = tv_elapsed (add_files_state.tv_start,
519 printf ("Processed %d %s in ", add_files_state.processed_files,
520 add_files_state.processed_files == 1 ?
521 "file" : "total files");
522 print_formatted_seconds (elapsed);
524 printf (" (%d files/sec.). \n",
525 (int) (add_files_state.processed_files / elapsed));
529 if (add_files_state.added_messages) {
530 printf ("Added %d %s to the database.\n\n",
531 add_files_state.added_messages,
532 add_files_state.added_messages == 1 ?
533 "message" : "unique messages");
536 printf ("When new mail is delivered to %s in the future,\n"
537 "run \"notmuch new\" to add it to the database.\n\n",
541 printf ("Note: At least one error was encountered: %s\n",
542 notmuch_status_to_string (ret));
547 free (mail_directory);
549 notmuch_database_close (notmuch);
555 tag_inbox_and_unread (notmuch_message_t *message)
557 notmuch_message_add_tag (message, "inbox");
558 notmuch_message_add_tag (message, "unread");
562 new_command (unused (int argc), unused (char *argv[]))
564 notmuch_database_t *notmuch;
565 const char *mail_directory;
566 add_files_state_t add_files_state;
568 struct timeval tv_now;
571 notmuch = notmuch_database_open (NULL);
572 if (notmuch == NULL) {
577 mail_directory = notmuch_database_get_path (notmuch);
579 add_files_state.ignore_read_only_directories = TRUE;
580 add_files_state.saw_read_only_directory = FALSE;
581 add_files_state.total_files = 0;
582 add_files_state.processed_files = 0;
583 add_files_state.added_messages = 0;
584 add_files_state.callback = tag_inbox_and_unread;
585 gettimeofday (&add_files_state.tv_start, NULL);
587 ret = add_files (notmuch, mail_directory, &add_files_state);
589 gettimeofday (&tv_now, NULL);
590 elapsed = tv_elapsed (add_files_state.tv_start,
592 if (add_files_state.processed_files) {
593 printf ("Processed %d %s in ", add_files_state.processed_files,
594 add_files_state.processed_files == 1 ?
595 "file" : "total files");
596 print_formatted_seconds (elapsed);
598 printf (" (%d files/sec.). \n",
599 (int) (add_files_state.processed_files / elapsed));
604 if (add_files_state.added_messages) {
605 printf ("Added %d new %s to the database (not much, really).\n",
606 add_files_state.added_messages,
607 add_files_state.added_messages == 1 ?
608 "message" : "messages");
610 printf ("No new mail---and that's not much.\n");
613 if (elapsed > 1 && ! add_files_state.saw_read_only_directory) {
614 printf ("\nTip: If you have any sub-directories that are archives (that is,\n"
615 "they will never receive new mail), marking these directores as\n"
616 "read-only (chmod u-w /path/to/dir) will make \"notmuch new\"\n"
617 "much more efficient (it won't even look in those directories).\n");
621 printf ("\nNote: At least one error was encountered: %s\n",
622 notmuch_status_to_string (ret));
627 notmuch_database_close (notmuch);
633 search_command (int argc, char *argv[])
635 void *local = talloc_new (NULL);
636 notmuch_database_t *notmuch = NULL;
637 notmuch_query_t *query;
638 notmuch_thread_results_t *results;
639 notmuch_thread_t *thread;
640 notmuch_tags_t *tags;
643 notmuch_status_t ret = NOTMUCH_STATUS_SUCCESS;
645 notmuch = notmuch_database_open (NULL);
646 if (notmuch == NULL) {
651 /* XXX: Should add xtalloc wrappers here and use them. */
652 query_str = talloc_strdup (local, "");
654 for (i = 0; i < argc; i++) {
656 query_str = talloc_asprintf_append (query_str, " ");
658 query_str = talloc_asprintf_append (query_str, "%s", argv[i]);
661 query = notmuch_query_create (notmuch, query_str);
663 fprintf (stderr, "Out of memory\n");
668 for (results = notmuch_query_search_threads (query);
669 notmuch_thread_results_has_more (results);
670 notmuch_thread_results_advance (results))
674 thread = notmuch_thread_results_get (results);
677 notmuch_thread_get_thread_id (thread),
678 notmuch_thread_get_subject (thread));
681 for (tags = notmuch_thread_get_tags (thread);
682 notmuch_tags_has_more (tags);
683 notmuch_tags_advance (tags))
687 printf ("%s", notmuch_tags_get (tags));
692 notmuch_thread_destroy (thread);
695 notmuch_query_destroy (query);
699 notmuch_database_close (notmuch);
706 show_command (unused (int argc), unused (char *argv[]))
708 fprintf (stderr, "Error: show is not implemented yet.\n");
713 dump_command (int argc, char *argv[])
716 notmuch_database_t *notmuch = NULL;
717 notmuch_query_t *query;
718 notmuch_message_results_t *results;
719 notmuch_message_t *message;
720 notmuch_tags_t *tags;
724 output = fopen (argv[0], "w");
725 if (output == NULL) {
726 fprintf (stderr, "Error opening %s for writing: %s\n",
727 argv[0], strerror (errno));
735 notmuch = notmuch_database_open (NULL);
736 if (notmuch == NULL) {
741 query = notmuch_query_create (notmuch, "");
743 fprintf (stderr, "Out of memory\n");
748 notmuch_query_set_sort (query, NOTMUCH_SORT_MESSAGE_ID);
750 for (results = notmuch_query_search_messages (query);
751 notmuch_message_results_has_more (results);
752 notmuch_message_results_advance (results))
755 message = notmuch_message_results_get (results);
758 "%s (", notmuch_message_get_message_id (message));
760 for (tags = notmuch_message_get_tags (message);
761 notmuch_tags_has_more (tags);
762 notmuch_tags_advance (tags))
765 fprintf (output, " ");
767 fprintf (output, "%s", notmuch_tags_get (tags));
772 fprintf (output, ")\n");
774 notmuch_message_destroy (message);
777 notmuch_query_destroy (query);
781 notmuch_database_close (notmuch);
782 if (output && output != stdout)
789 restore_command (int argc, char *argv[])
792 notmuch_database_t *notmuch = NULL;
801 input = fopen (argv[0], "r");
803 fprintf (stderr, "Error opening %s for reading: %s\n",
804 argv[0], strerror (errno));
809 printf ("No filename given. Reading dump from stdin.\n");
813 notmuch = notmuch_database_open (NULL);
814 if (notmuch == NULL) {
819 /* Dump output is one line per message. We match a sequence of
820 * non-space characters for the message-id, then one or more
821 * spaces, then a list of space-separated tags as a sequence of
822 * characters within literal '(' and ')'. */
824 "^([^ ]+) \\(([^)]*)\\)$",
827 while ((line_len = getline (&line, &line_size, input)) != -1) {
829 char *message_id, *tags, *tag, *next;
830 notmuch_message_t *message;
831 notmuch_status_t status;
833 chomp_newline (line);
835 rerr = xregexec (®ex, line, 3, match, 0);
836 if (rerr == REG_NOMATCH)
838 fprintf (stderr, "Warning: Ignoring invalid input line: %s\n",
843 message_id = xstrndup (line + match[1].rm_so,
844 match[1].rm_eo - match[1].rm_so);
845 tags = xstrndup (line + match[2].rm_so,
846 match[2].rm_eo - match[2].rm_so);
850 message = notmuch_database_find_message (notmuch, message_id);
851 if (message == NULL) {
852 fprintf (stderr, "Warning: Cannot apply tags to missing message: %s (",
858 tag = strsep (&next, " ");
862 status = notmuch_message_add_tag (message, tag);
865 "Error applying tag %s to message %s:\n",
867 fprintf (stderr, "%s\n",
868 notmuch_status_to_string (status));
871 fprintf (stderr, "%s%s",
872 tag == tags ? "" : " ", tag);
877 notmuch_message_destroy (message);
879 fprintf (stderr, ")\n");
891 notmuch_database_close (notmuch);
892 if (input && input != stdin)
898 command_t commands[] = {
899 { "setup", setup_command,
900 "Interactively setup notmuch for first use.\n\n"
901 "\t\tInvoking notmuch with no command argument will run setup if\n"
902 "\t\tthe setup command has not previously been completed." },
903 { "new", new_command,
904 "Find and import any new messages.\n\n"
905 "\t\tScans all sub-directories of the database, adding new messages\n"
906 "\t\tthat are found. Each new message will be tagges as both\n"
907 "\t\t\"inbox\" and \"unread\".\n"
909 "\t\tNote: \"notmuch new\" will skip any read-only directories,\n"
910 "\t\tso you can use that to mark tdirectories that will not\n"
911 "\t\treceive any new mail (and make \"notmuch new\" faster)." },
912 { "search", search_command,
913 "<search-term> [...]\n\n"
914 "\t\tSearch for threads matching the given search terms.\n"
915 "\t\tOnce we actually implement search we'll document the\n"
916 "\t\tsyntax here." },
917 { "show", show_command,
919 "\t\tShow the thread with the given thread ID (see 'search')." },
920 { "dump", dump_command,
922 "\t\tCreate a plain-text dump of the tags for each message\n"
923 "\t\twriting to the given filename, if any, or to stdout.\n"
924 "\t\tThese tags are the only data in the notmuch database\n"
925 "\t\tthat can't be recreated from the messages themselves.\n"
926 "\t\tThe output of notmuch dump is therefore the only\n"
927 "\t\tcritical thing to backup (and much more friendly to\n"
928 "\t\tincremental backup than the native database files." },
929 { "restore", restore_command,
931 "\t\tRestore the tags from the given dump file (see 'dump')." }
940 fprintf (stderr, "Usage: notmuch <command> [args...]\n");
941 fprintf (stderr, "\n");
942 fprintf (stderr, "Where <command> and [args...] are as follows:\n");
943 fprintf (stderr, "\n");
945 for (i = 0; i < ARRAY_SIZE (commands); i++) {
946 command = &commands[i];
948 fprintf (stderr, "\t%s\t%s\n\n", command->name, command->usage);
953 main (int argc, char *argv[])
959 return setup_command (0, NULL);
961 for (i = 0; i < ARRAY_SIZE (commands); i++) {
962 command = &commands[i];
964 if (strcmp (argv[1], command->name) == 0)
965 return (command->function) (argc - 2, &argv[2]);
968 /* Don't complain about "help" being an unknown command when we're
969 about to provide exactly what's wanted anyway. */
970 if (strcmp (argv[1], "help") == 0 ||
971 strcmp (argv[1], "--help") == 0)
973 fprintf (stderr, "The notmuch mail system.\n\n");
977 fprintf (stderr, "Error: Unknown command '%s'\n\n", argv[1]);