1 /* notmuch - Not much of an email program, (just index and search)
3 * Copyright © 2009 Carl Worth
5 * This program is free software: you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation, either version 3 of the License, or
8 * (at your option) any later version.
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
15 * You should have received a copy of the GNU General Public License
16 * along with this program. If not, see http://www.gnu.org/licenses/ .
18 * Author: Carl Worth <cworth@cworth.org>
22 #define _GNU_SOURCE /* for getline */
28 /* This is separate from notmuch-private.h because we're trying to
29 * keep notmuch.c from looking into any internals, (which helps us
30 * develop notmuch.h into a plausible library interface).
44 #include <glib.h> /* g_strdup_printf */
46 #define unused(x) x __attribute__ ((unused))
48 /* There's no point in continuing when we've detected that we've done
49 * something wrong internally (as opposed to the user passing in a
52 * Note that __location__ comes from talloc.h.
54 #define INTERNAL_ERROR(format, ...) \
57 "Internal error: " format " (%s)\n", \
58 ##__VA_ARGS__, __location__); \
62 #define ARRAY_SIZE(arr) (sizeof (arr) / sizeof (arr[0]))
64 typedef int (*command_function_t) (int argc, char *argv[]);
66 typedef struct command {
68 command_function_t function;
73 int ignore_read_only_directories;
74 int saw_read_only_directory;
79 struct timeval tv_start;
83 chomp_newline (char *str)
85 if (str && str[strlen(str)-1] == '\n')
86 str[strlen(str)-1] = '\0';
89 /* Compute the number of seconds elapsed from start to end. */
91 tv_elapsed (struct timeval start, struct timeval end)
93 return ((end.tv_sec - start.tv_sec) +
94 (end.tv_usec - start.tv_usec) / 1e6);
98 print_formatted_seconds (double seconds)
104 printf ("almost no time");
108 if (seconds > 3600) {
109 hours = (int) seconds / 3600;
110 printf ("%dh ", hours);
111 seconds -= hours * 3600;
115 minutes = (int) seconds / 60;
116 printf ("%dm ", minutes);
117 seconds -= minutes * 60;
120 printf ("%ds", (int) seconds);
124 add_files_print_progress (add_files_state_t *state)
126 struct timeval tv_now;
127 double elapsed_overall, rate_overall;
129 gettimeofday (&tv_now, NULL);
131 elapsed_overall = tv_elapsed (state->tv_start, tv_now);
132 rate_overall = (state->processed_files) / elapsed_overall;
134 printf ("Processed %d", state->processed_files);
136 if (state->total_files) {
137 printf (" of %d files (", state->total_files);
138 print_formatted_seconds ((state->total_files - state->processed_files) /
140 printf (" remaining). \r");
142 printf (" files (%d files/sec.) \r", (int) rate_overall);
148 /* Examine 'path' recursively as follows:
150 * o Ask the filesystem for the mtime of 'path' (path_mtime)
152 * o Ask the database for its timestamp of 'path' (path_dbtime)
154 * o If 'path_mtime' > 'path_dbtime'
156 * o For each regular file in 'path' with mtime newer than the
157 * 'path_dbtime' call add_message to add the file to the
160 * o For each sub-directory of path, recursively call into this
163 * o Tell the database to update its time of 'path' to 'path_mtime'
165 * The 'struct stat *st' must point to a structure that has already
166 * been initialized for 'path' by calling stat().
168 static notmuch_status_t
169 add_files_recursive (notmuch_database_t *notmuch,
172 add_files_state_t *state)
175 struct dirent *e, *entry = NULL;
179 time_t path_mtime, path_dbtime;
180 notmuch_status_t status, ret = NOTMUCH_STATUS_SUCCESS;
182 /* If we're told to, we bail out on encountering a read-only
183 * directory, (with this being a clear clue from the user to
184 * Notmuch that new mail won't be arriving there and we need not
186 if (state->ignore_read_only_directories &&
187 (st->st_mode & S_IWUSR) == 0)
189 state->saw_read_only_directory = TRUE;
193 path_mtime = st->st_mtime;
195 path_dbtime = notmuch_database_get_timestamp (notmuch, path);
197 dir = opendir (path);
199 fprintf (stderr, "Error opening directory %s: %s\n",
200 path, strerror (errno));
201 ret = NOTMUCH_STATUS_FILE_ERROR;
205 entry_length = offsetof (struct dirent, d_name) +
206 pathconf (path, _PC_NAME_MAX) + 1;
207 entry = malloc (entry_length);
210 err = readdir_r (dir, entry, &e);
212 fprintf (stderr, "Error reading directory: %s\n",
214 ret = NOTMUCH_STATUS_FILE_ERROR;
221 /* If this directory hasn't been modified since the last
222 * add_files, then we only need to look further for
223 * sub-directories. */
224 if (path_mtime <= path_dbtime && entry->d_type != DT_DIR)
227 /* Ignore special directories to avoid infinite recursion.
228 * Also ignore the .notmuch directory.
230 /* XXX: Eventually we'll want more sophistication to let the
231 * user specify files to be ignored. */
232 if (strcmp (entry->d_name, ".") == 0 ||
233 strcmp (entry->d_name, "..") == 0 ||
234 strcmp (entry->d_name, ".notmuch") ==0)
239 next = g_strdup_printf ("%s/%s", path, entry->d_name);
241 if (stat (next, st)) {
242 fprintf (stderr, "Error reading %s: %s\n",
243 next, strerror (errno));
244 ret = NOTMUCH_STATUS_FILE_ERROR;
248 if (S_ISREG (st->st_mode)) {
249 /* If the file hasn't been modified since the last
250 * add_files, then we need not look at it. */
251 if (st->st_mtime > path_dbtime) {
252 state->processed_files++;
254 status = notmuch_database_add_message (notmuch, next);
257 case NOTMUCH_STATUS_SUCCESS:
258 state->added_messages++;
260 /* Non-fatal issues (go on to next file) */
261 case NOTMUCH_STATUS_DUPLICATE_MESSAGE_ID:
262 /* Stay silent on this one. */
264 case NOTMUCH_STATUS_FILE_NOT_EMAIL:
265 fprintf (stderr, "Note: Ignoring non-mail file: %s\n",
268 /* Fatal issues. Don't process anymore. */
269 case NOTMUCH_STATUS_XAPIAN_EXCEPTION:
270 case NOTMUCH_STATUS_OUT_OF_MEMORY:
271 fprintf (stderr, "Error: %s. Halting processing.\n",
272 notmuch_status_to_string (status));
276 case NOTMUCH_STATUS_FILE_ERROR:
277 case NOTMUCH_STATUS_NULL_POINTER:
278 case NOTMUCH_STATUS_TAG_TOO_LONG:
279 case NOTMUCH_STATUS_LAST_STATUS:
280 INTERNAL_ERROR ("add_message returned unexpected value: %d", status);
283 if (state->processed_files % 1000 == 0)
284 add_files_print_progress (state);
286 } else if (S_ISDIR (st->st_mode)) {
287 status = add_files_recursive (notmuch, next, st, state);
288 if (status && ret == NOTMUCH_STATUS_SUCCESS)
296 status = notmuch_database_set_timestamp (notmuch, path, path_mtime);
297 if (status && ret == NOTMUCH_STATUS_SUCCESS)
311 /* This is the top-level entry point for add_files. It does a couple
312 * of error checks, and then calls into the recursive function,
313 * (avoiding the repeating of these error checks at every
314 * level---which would be useless becaues we already do a stat() at
315 * the level above). */
316 static notmuch_status_t
317 add_files (notmuch_database_t *notmuch,
319 add_files_state_t *state)
323 if (stat (path, &st)) {
324 fprintf (stderr, "Error reading directory %s: %s\n",
325 path, strerror (errno));
326 return NOTMUCH_STATUS_FILE_ERROR;
329 if (! S_ISDIR (st.st_mode)) {
330 fprintf (stderr, "Error: %s is not a directory.\n", path);
331 return NOTMUCH_STATUS_FILE_ERROR;
334 return add_files_recursive (notmuch, path, &st, state);
337 /* Recursively count all regular files in path and all sub-direcotries
338 * of path. The result is added to *count (which should be
339 * initialized to zero by the top-level caller before calling
342 count_files (const char *path, int *count)
345 struct dirent *entry, *e;
351 dir = opendir (path);
354 fprintf (stderr, "Warning: failed to open directory %s: %s\n",
355 path, strerror (errno));
359 entry_length = offsetof (struct dirent, d_name) +
360 pathconf (path, _PC_NAME_MAX) + 1;
361 entry = malloc (entry_length);
364 err = readdir_r (dir, entry, &e);
366 fprintf (stderr, "Error reading directory: %s\n",
375 /* Ignore special directories to avoid infinite recursion.
376 * Also ignore the .notmuch directory.
378 /* XXX: Eventually we'll want more sophistication to let the
379 * user specify files to be ignored. */
380 if (strcmp (entry->d_name, ".") == 0 ||
381 strcmp (entry->d_name, "..") == 0 ||
382 strcmp (entry->d_name, ".notmuch") == 0)
387 next = g_strdup_printf ("%s/%s", path, entry->d_name);
391 if (S_ISREG (st.st_mode)) {
393 if (*count % 1000 == 0) {
394 printf ("Found %d files so far.\r", *count);
397 } else if (S_ISDIR (st.st_mode)) {
398 count_files (next, count);
410 setup_command (unused (int argc), unused (char *argv[]))
412 notmuch_database_t *notmuch = NULL;
413 char *default_path, *mail_directory = NULL;
416 add_files_state_t add_files_state;
418 struct timeval tv_now;
419 notmuch_status_t ret = NOTMUCH_STATUS_SUCCESS;
421 printf ("Welcome to notmuch!\n\n");
423 printf ("The goal of notmuch is to help you manage and search your collection of\n"
424 "email, and to efficiently keep up with the flow of email as it comes in.\n\n");
426 printf ("Notmuch needs to know the top-level directory of your email archive,\n"
427 "(where you already have mail stored and where messages will be delivered\n"
428 "in the future). This directory can contain any number of sub-directories\n"
429 "and primarily just files with indvidual email messages (eg. maildir or mh\n"
430 "archives are perfect). If there are other, non-email files (such as\n"
431 "indexes maintained by other email programs) then notmuch will do its\n"
432 "best to detect those and ignore them.\n\n");
434 printf ("Mail storage that uses mbox format, (where one mbox file contains many\n"
435 "messages), will not work with notmuch. If that's how your mail is currently\n"
436 "stored, we recommend you first convert it to maildir format with a utility\n"
437 "such as mb2md. In that case, press Control-C now and run notmuch again\n"
438 "once the conversion is complete.\n\n");
441 default_path = notmuch_database_default_path ();
442 printf ("Top-level mail directory [%s]: ", default_path);
445 getline (&mail_directory, &line_size, stdin);
446 chomp_newline (mail_directory);
450 if (mail_directory == NULL || strlen (mail_directory) == 0) {
452 free (mail_directory);
453 mail_directory = default_path;
455 /* XXX: Instead of telling the user to use an environment
456 * variable here, we should really be writing out a configuration
457 * file and loading that on the next run. */
458 if (strcmp (mail_directory, default_path)) {
459 printf ("Note: Since you are not using the default path, you will want to set\n"
460 "the NOTMUCH_BASE environment variable to %s so that\n"
461 "future calls to notmuch commands will know where to find your mail.\n",
463 printf ("For example, if you are using bash for your shell, add:\n\n");
464 printf ("\texport NOTMUCH_BASE=%s\n\n", mail_directory);
465 printf ("to your ~/.bashrc file.\n\n");
470 notmuch = notmuch_database_create (mail_directory);
471 if (notmuch == NULL) {
472 fprintf (stderr, "Failed to create new notmuch database at %s\n",
474 ret = NOTMUCH_STATUS_FILE_ERROR;
478 printf ("OK. Let's take a look at the mail we can find in the directory\n");
479 printf ("%s ...\n", mail_directory);
482 count_files (mail_directory, &count);
484 printf ("Found %d total files. That's not much mail.\n\n", count);
486 printf ("Next, we'll inspect the messages and create a database of threads:\n");
488 add_files_state.ignore_read_only_directories = FALSE;
489 add_files_state.saw_read_only_directory = FALSE;
490 add_files_state.total_files = count;
491 add_files_state.processed_files = 0;
492 add_files_state.added_messages = 0;
493 gettimeofday (&add_files_state.tv_start, NULL);
495 ret = add_files (notmuch, mail_directory, &add_files_state);
497 gettimeofday (&tv_now, NULL);
498 elapsed = tv_elapsed (add_files_state.tv_start,
500 printf ("Processed %d %s in ", add_files_state.processed_files,
501 add_files_state.processed_files == 1 ?
502 "file" : "total files");
503 print_formatted_seconds (elapsed);
505 printf (" (%d files/sec.). \n",
506 (int) (add_files_state.processed_files / elapsed));
510 if (add_files_state.added_messages) {
511 printf ("Added %d %s to the database.\n\n",
512 add_files_state.added_messages,
513 add_files_state.added_messages == 1 ?
514 "message" : "unique messages");
517 printf ("When new mail is delivered to %s in the future,\n"
518 "run \"notmuch new\" to add it to the database.\n\n",
522 printf ("Note: At least one error was encountered: %s\n",
523 notmuch_status_to_string (ret));
528 free (mail_directory);
530 notmuch_database_close (notmuch);
536 new_command (unused (int argc), unused (char *argv[]))
538 notmuch_database_t *notmuch;
539 const char *mail_directory;
540 add_files_state_t add_files_state;
542 struct timeval tv_now;
545 notmuch = notmuch_database_open (NULL);
546 if (notmuch == NULL) {
551 mail_directory = notmuch_database_get_path (notmuch);
553 add_files_state.ignore_read_only_directories = TRUE;
554 add_files_state.saw_read_only_directory = FALSE;
555 add_files_state.total_files = 0;
556 add_files_state.processed_files = 0;
557 add_files_state.added_messages = 0;
558 gettimeofday (&add_files_state.tv_start, NULL);
560 ret = add_files (notmuch, mail_directory, &add_files_state);
562 gettimeofday (&tv_now, NULL);
563 elapsed = tv_elapsed (add_files_state.tv_start,
565 if (add_files_state.processed_files) {
566 printf ("Processed %d %s in ", add_files_state.processed_files,
567 add_files_state.processed_files == 1 ?
568 "file" : "total files");
569 print_formatted_seconds (elapsed);
571 printf (" (%d files/sec.). \n",
572 (int) (add_files_state.processed_files / elapsed));
577 if (add_files_state.added_messages) {
578 printf ("Added %d new %s to the database (not much, really).\n",
579 add_files_state.added_messages,
580 add_files_state.added_messages == 1 ?
581 "message" : "messages");
583 printf ("No new mail---and that's not much.\n");
586 if (elapsed > 1 && ! add_files_state.saw_read_only_directory) {
587 printf ("\nTip: If you have any sub-directories that are archives (that is,\n"
588 "they will never receive new mail), marking these directores as\n"
589 "read-only (chmod u-w /path/to/dir) will make \"notmuch new\"\n"
590 "much more efficient (it won't even look in those directories).\n");
594 printf ("\nNote: At least one error was encountered: %s\n",
595 notmuch_status_to_string (ret));
600 notmuch_database_close (notmuch);
606 search_command (int argc, char *argv[])
608 void *local = talloc_new (NULL);
609 notmuch_database_t *notmuch = NULL;
610 notmuch_query_t *query;
611 notmuch_results_t *results;
612 notmuch_message_t *message;
613 notmuch_tags_t *tags;
616 notmuch_status_t ret = NOTMUCH_STATUS_SUCCESS;
618 notmuch = notmuch_database_open (NULL);
619 if (notmuch == NULL) {
624 /* XXX: Should add xtalloc wrappers here and use them. */
625 query_str = talloc_strdup (local, "");
627 for (i = 0; i < argc; i++) {
629 query_str = talloc_asprintf_append (query_str, " ");
631 query_str = talloc_asprintf_append (query_str, "%s", argv[i]);
634 query = notmuch_query_create (notmuch, query_str);
636 fprintf (stderr, "Out of memory\n");
641 for (results = notmuch_query_search (query);
642 notmuch_results_has_more (results);
643 notmuch_results_advance (results))
646 message = notmuch_results_get (results);
648 printf ("%s (", notmuch_message_get_message_id (message));
650 for (tags = notmuch_message_get_tags (message);
651 notmuch_tags_has_more (tags);
652 notmuch_tags_advance (tags))
657 printf ("%s", notmuch_tags_get (tags));
664 notmuch_message_destroy (message);
667 notmuch_query_destroy (query);
671 notmuch_database_close (notmuch);
678 show_command (unused (int argc), unused (char *argv[]))
680 fprintf (stderr, "Error: show is not implemented yet.\n");
685 dump_command (int argc, char *argv[])
688 notmuch_database_t *notmuch = NULL;
689 notmuch_query_t *query;
690 notmuch_results_t *results;
691 notmuch_message_t *message;
692 notmuch_tags_t *tags;
696 output = fopen (argv[0], "w");
697 if (output == NULL) {
698 fprintf (stderr, "Error opening %s for writing: %s\n",
699 argv[0], strerror (errno));
707 notmuch = notmuch_database_open (NULL);
708 if (notmuch == NULL) {
713 query = notmuch_query_create (notmuch, "");
715 fprintf (stderr, "Out of memory\n");
720 notmuch_query_set_sort (query, NOTMUCH_SORT_MESSAGE_ID);
722 for (results = notmuch_query_search (query);
723 notmuch_results_has_more (results);
724 notmuch_results_advance (results))
727 message = notmuch_results_get (results);
730 "%s (", notmuch_message_get_message_id (message));
732 for (tags = notmuch_message_get_tags (message);
733 notmuch_tags_has_more (tags);
734 notmuch_tags_advance (tags))
737 fprintf (output, " ");
739 fprintf (output, "%s", notmuch_tags_get (tags));
744 fprintf (output, ")\n");
746 notmuch_message_destroy (message);
749 notmuch_query_destroy (query);
753 notmuch_database_close (notmuch);
754 if (output != stdout)
761 restore_command (int argc, char *argv[])
764 notmuch_database_t *notmuch = NULL;
773 input = fopen (argv[0], "r");
775 fprintf (stderr, "Error opening %s for reading: %s\n",
776 argv[0], strerror (errno));
781 printf ("No filename given. Reading dump from stdin.\n");
785 notmuch = notmuch_database_open (NULL);
786 if (notmuch == NULL) {
791 /* Dump output is one line per message. We match a sequence of
792 * non-space characters for the message-id, then one or more
793 * spaces, then a list of space-separated tags as a sequence of
794 * characters within literal '(' and ')'. */
796 "^([^ ]+) \\(([^)]*)\\)$",
799 while ((line_len = getline (&line, &line_size, input)) != -1) {
801 char *message_id, *tags, *tag, *next;
802 notmuch_message_t *message;
803 notmuch_status_t status;
805 chomp_newline (line);
807 rerr = xregexec (®ex, line, 3, match, 0);
808 if (rerr == REG_NOMATCH)
810 fprintf (stderr, "Warning: Ignoring invalid input line: %s\n",
815 message_id = xstrndup (line + match[1].rm_so,
816 match[1].rm_eo - match[1].rm_so);
817 tags = xstrndup (line + match[2].rm_so,
818 match[2].rm_eo - match[2].rm_so);
822 message = notmuch_database_find_message (notmuch, message_id);
823 if (message == NULL) {
824 fprintf (stderr, "Warning: Cannot apply tags to missing message: %s (",
830 tag = strsep (&next, " ");
834 status = notmuch_message_add_tag (message, tag);
837 "Error applying tag %s to message %s:\n",
839 fprintf (stderr, "%s\n",
840 notmuch_status_to_string (status));
843 fprintf (stderr, "%s%s",
844 tag == tags ? "" : " ", tag);
849 notmuch_message_destroy (message);
851 fprintf (stderr, ")\n");
863 notmuch_database_close (notmuch);
868 command_t commands[] = {
869 { "setup", setup_command,
870 "Interactively setup notmuch for first use.\n\n"
871 "\t\tInvoking notmuch with no command argument will run setup if\n"
872 "\t\tthe setup command has not previously been completed." },
873 { "new", new_command,
874 "Find and import any new messages.\n\n"
875 "\t\tScans all sub-directories of the database, adding new files\n"
876 "\t\tthat are found. Note: \"notmuch new\" will skip any\n"
877 "\t\tread-only directories, so you can use that to mark\n"
878 "\t\tdirectories that will not receive any new mail."},
879 { "search", search_command,
880 "<search-term> [...]\n\n"
881 "\t\tSearch for threads matching the given search terms.\n"
882 "\t\tOnce we actually implement search we'll document the\n"
883 "\t\tsyntax here." },
884 { "show", show_command,
886 "\t\tShow the thread with the given thread ID (see 'search')." },
887 { "dump", dump_command,
889 "\t\tCreate a plain-text dump of the tags for each message\n"
890 "\t\twriting to the given filename, if any, or to stdout.\n"
891 "\t\tThese tags are the only data in the notmuch database\n"
892 "\t\tthat can't be recreated from the messages themselves.\n"
893 "\t\tThe output of notmuch dump is therefore the only\n"
894 "\t\tcritical thing to backup (and much more friendly to\n"
895 "\t\tincremental backup than the native database files." },
896 { "restore", restore_command,
898 "\t\tRestore the tags from the given dump file (see 'dump')." }
907 fprintf (stderr, "Usage: notmuch <command> [args...]\n");
908 fprintf (stderr, "\n");
909 fprintf (stderr, "Where <command> and [args...] are as follows:\n");
910 fprintf (stderr, "\n");
912 for (i = 0; i < ARRAY_SIZE (commands); i++) {
913 command = &commands[i];
915 fprintf (stderr, "\t%s\t%s\n\n", command->name, command->usage);
920 main (int argc, char *argv[])
926 return setup_command (0, NULL);
928 for (i = 0; i < ARRAY_SIZE (commands); i++) {
929 command = &commands[i];
931 if (strcmp (argv[1], command->name) == 0)
932 return (command->function) (argc - 2, &argv[2]);
935 /* Don't complain about "help" being an unknown command when we're
936 about to provide exactly what's wanted anyway. */
937 if (strcmp (argv[1], "help") == 0 ||
938 strcmp (argv[1], "--help") == 0)
940 fprintf (stderr, "The notmuch mail system.\n\n");
942 fprintf (stderr, "Error: Unknown command '%s'\n\n", argv[1]);