1 /* notmuch - Not much of an email program, (just index and search)
3 * Copyright © 2009 Carl Worth
5 * This program is free software: you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation, either version 3 of the License, or
8 * (at your option) any later version.
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
15 * You should have received a copy of the GNU General Public License
16 * along with this program. If not, see http://www.gnu.org/licenses/ .
18 * Author: Carl Worth <cworth@cworth.org>
22 #define _GNU_SOURCE /* for getline */
28 /* This is separate from notmuch-private.h because we're trying to
29 * keep notmuch.c from looking into any internals, (which helps us
30 * develop notmuch.h into a plausible library interface).
44 #define unused(x) x __attribute__ ((unused))
46 /* There's no point in continuing when we've detected that we've done
47 * something wrong internally (as opposed to the user passing in a
50 * Note that __location__ comes from talloc.h.
52 #define INTERNAL_ERROR(format, ...) \
55 "Internal error: " format " (%s)\n", \
56 ##__VA_ARGS__, __location__); \
60 #define ARRAY_SIZE(arr) (sizeof (arr) / sizeof (arr[0]))
62 typedef int (*command_function_t) (int argc, char *argv[]);
64 typedef struct command {
66 command_function_t function;
70 typedef void (*add_files_callback_t) (notmuch_message_t *message);
73 int ignore_read_only_directories;
74 int saw_read_only_directory;
79 struct timeval tv_start;
81 add_files_callback_t callback;
85 chomp_newline (char *str)
87 if (str && str[strlen(str)-1] == '\n')
88 str[strlen(str)-1] = '\0';
91 /* Compute the number of seconds elapsed from start to end. */
93 tv_elapsed (struct timeval start, struct timeval end)
95 return ((end.tv_sec - start.tv_sec) +
96 (end.tv_usec - start.tv_usec) / 1e6);
100 print_formatted_seconds (double seconds)
106 printf ("almost no time");
110 if (seconds > 3600) {
111 hours = (int) seconds / 3600;
112 printf ("%dh ", hours);
113 seconds -= hours * 3600;
117 minutes = (int) seconds / 60;
118 printf ("%dm ", minutes);
119 seconds -= minutes * 60;
122 printf ("%ds", (int) seconds);
126 add_files_print_progress (add_files_state_t *state)
128 struct timeval tv_now;
129 double elapsed_overall, rate_overall;
131 gettimeofday (&tv_now, NULL);
133 elapsed_overall = tv_elapsed (state->tv_start, tv_now);
134 rate_overall = (state->processed_files) / elapsed_overall;
136 printf ("Processed %d", state->processed_files);
138 if (state->total_files) {
139 printf (" of %d files (", state->total_files);
140 print_formatted_seconds ((state->total_files - state->processed_files) /
142 printf (" remaining). \r");
144 printf (" files (%d files/sec.) \r", (int) rate_overall);
150 /* Examine 'path' recursively as follows:
152 * o Ask the filesystem for the mtime of 'path' (path_mtime)
154 * o Ask the database for its timestamp of 'path' (path_dbtime)
156 * o If 'path_mtime' > 'path_dbtime'
158 * o For each regular file in 'path' with mtime newer than the
159 * 'path_dbtime' call add_message to add the file to the
162 * o For each sub-directory of path, recursively call into this
165 * o Tell the database to update its time of 'path' to 'path_mtime'
167 * The 'struct stat *st' must point to a structure that has already
168 * been initialized for 'path' by calling stat().
170 static notmuch_status_t
171 add_files_recursive (notmuch_database_t *notmuch,
174 add_files_state_t *state)
177 struct dirent *e, *entry = NULL;
181 time_t path_mtime, path_dbtime;
182 notmuch_status_t status, ret = NOTMUCH_STATUS_SUCCESS;
183 notmuch_message_t *message, **closure;
185 /* If we're told to, we bail out on encountering a read-only
186 * directory, (with this being a clear clue from the user to
187 * Notmuch that new mail won't be arriving there and we need not
189 if (state->ignore_read_only_directories &&
190 (st->st_mode & S_IWUSR) == 0)
192 state->saw_read_only_directory = TRUE;
196 path_mtime = st->st_mtime;
198 path_dbtime = notmuch_database_get_timestamp (notmuch, path);
200 dir = opendir (path);
202 fprintf (stderr, "Error opening directory %s: %s\n",
203 path, strerror (errno));
204 ret = NOTMUCH_STATUS_FILE_ERROR;
208 entry_length = offsetof (struct dirent, d_name) +
209 pathconf (path, _PC_NAME_MAX) + 1;
210 entry = malloc (entry_length);
213 err = readdir_r (dir, entry, &e);
215 fprintf (stderr, "Error reading directory: %s\n",
217 ret = NOTMUCH_STATUS_FILE_ERROR;
224 /* If this directory hasn't been modified since the last
225 * add_files, then we only need to look further for
226 * sub-directories. */
227 if (path_mtime <= path_dbtime && entry->d_type != DT_DIR)
230 /* Ignore special directories to avoid infinite recursion.
231 * Also ignore the .notmuch directory.
233 /* XXX: Eventually we'll want more sophistication to let the
234 * user specify files to be ignored. */
235 if (strcmp (entry->d_name, ".") == 0 ||
236 strcmp (entry->d_name, "..") == 0 ||
237 strcmp (entry->d_name, ".notmuch") ==0)
242 next = talloc_asprintf (notmuch, "%s/%s", path, entry->d_name);
244 if (stat (next, st)) {
245 fprintf (stderr, "Error reading %s: %s\n",
246 next, strerror (errno));
247 ret = NOTMUCH_STATUS_FILE_ERROR;
251 if (S_ISREG (st->st_mode)) {
252 /* If the file hasn't been modified since the last
253 * add_files, then we need not look at it. */
254 if (st->st_mtime > path_dbtime) {
255 state->processed_files++;
262 status = notmuch_database_add_message (notmuch, next, closure);
265 case NOTMUCH_STATUS_SUCCESS:
266 state->added_messages++;
268 (state->callback) (message);
269 notmuch_message_destroy (message);
271 /* Non-fatal issues (go on to next file) */
272 case NOTMUCH_STATUS_DUPLICATE_MESSAGE_ID:
273 /* Stay silent on this one. */
275 case NOTMUCH_STATUS_FILE_NOT_EMAIL:
276 fprintf (stderr, "Note: Ignoring non-mail file: %s\n",
279 /* Fatal issues. Don't process anymore. */
280 case NOTMUCH_STATUS_XAPIAN_EXCEPTION:
281 case NOTMUCH_STATUS_OUT_OF_MEMORY:
282 fprintf (stderr, "Error: %s. Halting processing.\n",
283 notmuch_status_to_string (status));
287 case NOTMUCH_STATUS_FILE_ERROR:
288 case NOTMUCH_STATUS_NULL_POINTER:
289 case NOTMUCH_STATUS_TAG_TOO_LONG:
290 case NOTMUCH_STATUS_UNBALANCED_FREEZE_THAW:
291 case NOTMUCH_STATUS_LAST_STATUS:
292 INTERNAL_ERROR ("add_message returned unexpected value: %d", status);
295 if (state->processed_files % 1000 == 0)
296 add_files_print_progress (state);
298 } else if (S_ISDIR (st->st_mode)) {
299 status = add_files_recursive (notmuch, next, st, state);
300 if (status && ret == NOTMUCH_STATUS_SUCCESS)
308 status = notmuch_database_set_timestamp (notmuch, path, path_mtime);
309 if (status && ret == NOTMUCH_STATUS_SUCCESS)
323 /* This is the top-level entry point for add_files. It does a couple
324 * of error checks, and then calls into the recursive function,
325 * (avoiding the repeating of these error checks at every
326 * level---which would be useless becaues we already do a stat() at
327 * the level above). */
328 static notmuch_status_t
329 add_files (notmuch_database_t *notmuch,
331 add_files_state_t *state)
335 if (stat (path, &st)) {
336 fprintf (stderr, "Error reading directory %s: %s\n",
337 path, strerror (errno));
338 return NOTMUCH_STATUS_FILE_ERROR;
341 if (! S_ISDIR (st.st_mode)) {
342 fprintf (stderr, "Error: %s is not a directory.\n", path);
343 return NOTMUCH_STATUS_FILE_ERROR;
346 return add_files_recursive (notmuch, path, &st, state);
349 /* Recursively count all regular files in path and all sub-direcotries
350 * of path. The result is added to *count (which should be
351 * initialized to zero by the top-level caller before calling
354 count_files (const char *path, int *count)
357 struct dirent *e, *entry = NULL;
363 dir = opendir (path);
366 fprintf (stderr, "Warning: failed to open directory %s: %s\n",
367 path, strerror (errno));
371 entry_length = offsetof (struct dirent, d_name) +
372 pathconf (path, _PC_NAME_MAX) + 1;
373 entry = malloc (entry_length);
376 err = readdir_r (dir, entry, &e);
378 fprintf (stderr, "Error reading directory: %s\n",
387 /* Ignore special directories to avoid infinite recursion.
388 * Also ignore the .notmuch directory.
390 /* XXX: Eventually we'll want more sophistication to let the
391 * user specify files to be ignored. */
392 if (strcmp (entry->d_name, ".") == 0 ||
393 strcmp (entry->d_name, "..") == 0 ||
394 strcmp (entry->d_name, ".notmuch") == 0)
399 if (asprintf (&next, "%s/%s", path, entry->d_name) == -1) {
401 fprintf (stderr, "Error descending from %s to %s: Out of memory\n",
402 path, entry->d_name);
408 if (S_ISREG (st.st_mode)) {
410 if (*count % 1000 == 0) {
411 printf ("Found %d files so far.\r", *count);
414 } else if (S_ISDIR (st.st_mode)) {
415 count_files (next, count);
429 setup_command (unused (int argc), unused (char *argv[]))
431 notmuch_database_t *notmuch = NULL;
432 char *default_path, *mail_directory = NULL;
435 add_files_state_t add_files_state;
437 struct timeval tv_now;
438 notmuch_status_t ret = NOTMUCH_STATUS_SUCCESS;
440 printf ("Welcome to notmuch!\n\n");
442 printf ("The goal of notmuch is to help you manage and search your collection of\n"
443 "email, and to efficiently keep up with the flow of email as it comes in.\n\n");
445 printf ("Notmuch needs to know the top-level directory of your email archive,\n"
446 "(where you already have mail stored and where messages will be delivered\n"
447 "in the future). This directory can contain any number of sub-directories\n"
448 "and primarily just files with indvidual email messages (eg. maildir or mh\n"
449 "archives are perfect). If there are other, non-email files (such as\n"
450 "indexes maintained by other email programs) then notmuch will do its\n"
451 "best to detect those and ignore them.\n\n");
453 printf ("Mail storage that uses mbox format, (where one mbox file contains many\n"
454 "messages), will not work with notmuch. If that's how your mail is currently\n"
455 "stored, we recommend you first convert it to maildir format with a utility\n"
456 "such as mb2md. In that case, press Control-C now and run notmuch again\n"
457 "once the conversion is complete.\n\n");
460 default_path = notmuch_database_default_path ();
461 printf ("Top-level mail directory [%s]: ", default_path);
464 getline (&mail_directory, &line_size, stdin);
465 chomp_newline (mail_directory);
469 if (mail_directory == NULL || strlen (mail_directory) == 0) {
471 free (mail_directory);
472 mail_directory = default_path;
474 /* XXX: Instead of telling the user to use an environment
475 * variable here, we should really be writing out a configuration
476 * file and loading that on the next run. */
477 if (strcmp (mail_directory, default_path)) {
478 printf ("Note: Since you are not using the default path, you will want to set\n"
479 "the NOTMUCH_BASE environment variable to %s so that\n"
480 "future calls to notmuch commands will know where to find your mail.\n",
482 printf ("For example, if you are using bash for your shell, add:\n\n");
483 printf ("\texport NOTMUCH_BASE=%s\n\n", mail_directory);
484 printf ("to your ~/.bashrc file.\n\n");
489 notmuch = notmuch_database_create (mail_directory);
490 if (notmuch == NULL) {
491 fprintf (stderr, "Failed to create new notmuch database at %s\n",
493 ret = NOTMUCH_STATUS_FILE_ERROR;
497 printf ("OK. Let's take a look at the mail we can find in the directory\n");
498 printf ("%s ...\n", mail_directory);
501 count_files (mail_directory, &count);
503 printf ("Found %d total files. That's not much mail.\n\n", count);
505 printf ("Next, we'll inspect the messages and create a database of threads:\n");
507 add_files_state.ignore_read_only_directories = FALSE;
508 add_files_state.saw_read_only_directory = FALSE;
509 add_files_state.total_files = count;
510 add_files_state.processed_files = 0;
511 add_files_state.added_messages = 0;
512 add_files_state.callback = NULL;
513 gettimeofday (&add_files_state.tv_start, NULL);
515 ret = add_files (notmuch, mail_directory, &add_files_state);
517 gettimeofday (&tv_now, NULL);
518 elapsed = tv_elapsed (add_files_state.tv_start,
520 printf ("Processed %d %s in ", add_files_state.processed_files,
521 add_files_state.processed_files == 1 ?
522 "file" : "total files");
523 print_formatted_seconds (elapsed);
525 printf (" (%d files/sec.). \n",
526 (int) (add_files_state.processed_files / elapsed));
530 if (add_files_state.added_messages) {
531 printf ("Added %d %s to the database.\n\n",
532 add_files_state.added_messages,
533 add_files_state.added_messages == 1 ?
534 "message" : "unique messages");
537 printf ("When new mail is delivered to %s in the future,\n"
538 "run \"notmuch new\" to add it to the database.\n\n",
542 printf ("Note: At least one error was encountered: %s\n",
543 notmuch_status_to_string (ret));
548 free (mail_directory);
550 notmuch_database_close (notmuch);
556 tag_inbox_and_unread (notmuch_message_t *message)
558 notmuch_message_add_tag (message, "inbox");
559 notmuch_message_add_tag (message, "unread");
563 new_command (unused (int argc), unused (char *argv[]))
565 notmuch_database_t *notmuch;
566 const char *mail_directory;
567 add_files_state_t add_files_state;
569 struct timeval tv_now;
572 notmuch = notmuch_database_open (NULL);
573 if (notmuch == NULL) {
578 mail_directory = notmuch_database_get_path (notmuch);
580 add_files_state.ignore_read_only_directories = TRUE;
581 add_files_state.saw_read_only_directory = FALSE;
582 add_files_state.total_files = 0;
583 add_files_state.processed_files = 0;
584 add_files_state.added_messages = 0;
585 add_files_state.callback = tag_inbox_and_unread;
586 gettimeofday (&add_files_state.tv_start, NULL);
588 ret = add_files (notmuch, mail_directory, &add_files_state);
590 gettimeofday (&tv_now, NULL);
591 elapsed = tv_elapsed (add_files_state.tv_start,
593 if (add_files_state.processed_files) {
594 printf ("Processed %d %s in ", add_files_state.processed_files,
595 add_files_state.processed_files == 1 ?
596 "file" : "total files");
597 print_formatted_seconds (elapsed);
599 printf (" (%d files/sec.). \n",
600 (int) (add_files_state.processed_files / elapsed));
605 if (add_files_state.added_messages) {
606 printf ("Added %d new %s to the database (not much, really).\n",
607 add_files_state.added_messages,
608 add_files_state.added_messages == 1 ?
609 "message" : "messages");
611 printf ("No new mail---and that's not much.\n");
614 if (elapsed > 1 && ! add_files_state.saw_read_only_directory) {
615 printf ("\nTip: If you have any sub-directories that are archives (that is,\n"
616 "they will never receive new mail), marking these directores as\n"
617 "read-only (chmod u-w /path/to/dir) will make \"notmuch new\"\n"
618 "much more efficient (it won't even look in those directories).\n");
622 printf ("\nNote: At least one error was encountered: %s\n",
623 notmuch_status_to_string (ret));
628 notmuch_database_close (notmuch);
634 search_command (int argc, char *argv[])
636 void *local = talloc_new (NULL);
637 notmuch_database_t *notmuch = NULL;
638 notmuch_query_t *query;
639 notmuch_thread_results_t *results;
640 notmuch_thread_t *thread;
641 notmuch_tags_t *tags;
644 notmuch_status_t ret = NOTMUCH_STATUS_SUCCESS;
646 notmuch = notmuch_database_open (NULL);
647 if (notmuch == NULL) {
652 /* XXX: Should add xtalloc wrappers here and use them. */
653 query_str = talloc_strdup (local, "");
655 for (i = 0; i < argc; i++) {
657 query_str = talloc_asprintf_append (query_str, " ");
659 query_str = talloc_asprintf_append (query_str, "%s", argv[i]);
662 query = notmuch_query_create (notmuch, query_str);
664 fprintf (stderr, "Out of memory\n");
669 for (results = notmuch_query_search_threads (query);
670 notmuch_thread_results_has_more (results);
671 notmuch_thread_results_advance (results))
675 thread = notmuch_thread_results_get (results);
678 notmuch_thread_get_thread_id (thread),
679 notmuch_thread_get_subject (thread));
682 for (tags = notmuch_thread_get_tags (thread);
683 notmuch_tags_has_more (tags);
684 notmuch_tags_advance (tags))
688 printf ("%s", notmuch_tags_get (tags));
693 notmuch_thread_destroy (thread);
696 notmuch_query_destroy (query);
700 notmuch_database_close (notmuch);
707 show_command (unused (int argc), unused (char *argv[]))
709 fprintf (stderr, "Error: show is not implemented yet.\n");
714 dump_command (int argc, char *argv[])
717 notmuch_database_t *notmuch = NULL;
718 notmuch_query_t *query;
719 notmuch_message_results_t *results;
720 notmuch_message_t *message;
721 notmuch_tags_t *tags;
725 output = fopen (argv[0], "w");
726 if (output == NULL) {
727 fprintf (stderr, "Error opening %s for writing: %s\n",
728 argv[0], strerror (errno));
736 notmuch = notmuch_database_open (NULL);
737 if (notmuch == NULL) {
742 query = notmuch_query_create (notmuch, "");
744 fprintf (stderr, "Out of memory\n");
749 notmuch_query_set_sort (query, NOTMUCH_SORT_MESSAGE_ID);
751 for (results = notmuch_query_search_messages (query);
752 notmuch_message_results_has_more (results);
753 notmuch_message_results_advance (results))
756 message = notmuch_message_results_get (results);
759 "%s (", notmuch_message_get_message_id (message));
761 for (tags = notmuch_message_get_tags (message);
762 notmuch_tags_has_more (tags);
763 notmuch_tags_advance (tags))
766 fprintf (output, " ");
768 fprintf (output, "%s", notmuch_tags_get (tags));
773 fprintf (output, ")\n");
775 notmuch_message_destroy (message);
778 notmuch_query_destroy (query);
782 notmuch_database_close (notmuch);
783 if (output && output != stdout)
790 restore_command (int argc, char *argv[])
793 notmuch_database_t *notmuch = NULL;
802 input = fopen (argv[0], "r");
804 fprintf (stderr, "Error opening %s for reading: %s\n",
805 argv[0], strerror (errno));
810 printf ("No filename given. Reading dump from stdin.\n");
814 notmuch = notmuch_database_open (NULL);
815 if (notmuch == NULL) {
820 /* Dump output is one line per message. We match a sequence of
821 * non-space characters for the message-id, then one or more
822 * spaces, then a list of space-separated tags as a sequence of
823 * characters within literal '(' and ')'. */
825 "^([^ ]+) \\(([^)]*)\\)$",
828 while ((line_len = getline (&line, &line_size, input)) != -1) {
830 char *message_id, *tags, *tag, *next;
831 notmuch_message_t *message;
832 notmuch_status_t status;
834 chomp_newline (line);
836 rerr = xregexec (®ex, line, 3, match, 0);
837 if (rerr == REG_NOMATCH)
839 fprintf (stderr, "Warning: Ignoring invalid input line: %s\n",
844 message_id = xstrndup (line + match[1].rm_so,
845 match[1].rm_eo - match[1].rm_so);
846 tags = xstrndup (line + match[2].rm_so,
847 match[2].rm_eo - match[2].rm_so);
851 message = notmuch_database_find_message (notmuch, message_id);
852 if (message == NULL) {
853 fprintf (stderr, "Warning: Cannot apply tags to missing message: %s\n",
858 notmuch_message_freeze (message);
860 notmuch_message_remove_all_tags (message);
864 tag = strsep (&next, " ");
867 status = notmuch_message_add_tag (message, tag);
870 "Error applying tag %s to message %s:\n",
872 fprintf (stderr, "%s\n",
873 notmuch_status_to_string (status));
877 notmuch_message_thaw (message);
878 notmuch_message_destroy (message);
891 notmuch_database_close (notmuch);
892 if (input && input != stdin)
898 command_t commands[] = {
899 { "setup", setup_command,
900 "Interactively setup notmuch for first use.\n\n"
901 "\t\tInvoking notmuch with no command argument will run setup if\n"
902 "\t\tthe setup command has not previously been completed." },
903 { "new", new_command,
904 "Find and import any new messages.\n\n"
905 "\t\tScans all sub-directories of the database, adding new messages\n"
906 "\t\tthat are found. Each new message will be tagges as both\n"
907 "\t\t\"inbox\" and \"unread\".\n"
909 "\t\tNote: \"notmuch new\" will skip any read-only directories,\n"
910 "\t\tso you can use that to mark tdirectories that will not\n"
911 "\t\treceive any new mail (and make \"notmuch new\" faster)." },
912 { "search", search_command,
913 "<search-term> [...]\n\n"
914 "\t\tSearch for threads matching the given search terms.\n"
915 "\t\tOnce we actually implement search we'll document the\n"
916 "\t\tsyntax here." },
917 { "show", show_command,
919 "\t\tShow the thread with the given thread ID (see 'search')." },
920 { "dump", dump_command,
922 "\t\tCreate a plain-text dump of the tags for each message\n"
923 "\t\twriting to the given filename, if any, or to stdout.\n"
924 "\t\tThese tags are the only data in the notmuch database\n"
925 "\t\tthat can't be recreated from the messages themselves.\n"
926 "\t\tThe output of notmuch dump is therefore the only\n"
927 "\t\tcritical thing to backup (and much more friendly to\n"
928 "\t\tincremental backup than the native database files." },
929 { "restore", restore_command,
931 "\t\tRestore the tags from the given dump file (see 'dump')." }
940 fprintf (stderr, "Usage: notmuch <command> [args...]\n");
941 fprintf (stderr, "\n");
942 fprintf (stderr, "Where <command> and [args...] are as follows:\n");
943 fprintf (stderr, "\n");
945 for (i = 0; i < ARRAY_SIZE (commands); i++) {
946 command = &commands[i];
948 fprintf (stderr, "\t%s\t%s\n\n", command->name, command->usage);
953 main (int argc, char *argv[])
959 return setup_command (0, NULL);
961 for (i = 0; i < ARRAY_SIZE (commands); i++) {
962 command = &commands[i];
964 if (strcmp (argv[1], command->name) == 0)
965 return (command->function) (argc - 2, &argv[2]);
968 /* Don't complain about "help" being an unknown command when we're
969 about to provide exactly what's wanted anyway. */
970 if (strcmp (argv[1], "help") == 0 ||
971 strcmp (argv[1], "--help") == 0)
973 fprintf (stderr, "The notmuch mail system.\n\n");
977 fprintf (stderr, "Error: Unknown command '%s'\n\n", argv[1]);