1 /* notmuch - Not much of an email program, (just index and search)
3 * Copyright © 2009 Carl Worth
5 * This program is free software: you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation, either version 3 of the License, or
8 * (at your option) any later version.
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
15 * You should have received a copy of the GNU General Public License
16 * along with this program. If not, see http://www.gnu.org/licenses/ .
18 * Author: Carl Worth <cworth@cworth.org>
22 #define _GNU_SOURCE /* for getline */
28 /* This is separate from notmuch-private.h because we're trying to
29 * keep notmuch.c from looking into any internals, (which helps us
30 * develop notmuch.h into a plausible library interface).
44 #include <glib.h> /* g_strdup_printf */
46 #define ARRAY_SIZE(arr) (sizeof (arr) / sizeof (arr[0]))
48 typedef int (*command_function_t) (int argc, char *argv[]);
50 typedef struct command {
52 command_function_t function;
57 int ignore_read_only_directories;
58 int saw_read_only_directory;
63 struct timeval tv_start;
67 chomp_newline (char *str)
69 if (str && str[strlen(str)-1] == '\n')
70 str[strlen(str)-1] = '\0';
73 /* Compute the number of seconds elapsed from start to end. */
75 tv_elapsed (struct timeval start, struct timeval end)
77 return ((end.tv_sec - start.tv_sec) +
78 (end.tv_usec - start.tv_usec) / 1e6);
82 print_formatted_seconds (double seconds)
88 printf ("almost no time");
93 hours = (int) seconds / 3600;
94 printf ("%dh ", hours);
95 seconds -= hours * 3600;
99 minutes = (int) seconds / 60;
100 printf ("%dm ", minutes);
101 seconds -= minutes * 60;
104 printf ("%ds", (int) seconds);
108 add_files_print_progress (add_files_state_t *state)
110 struct timeval tv_now;
111 double elapsed_overall, rate_overall;
113 gettimeofday (&tv_now, NULL);
115 elapsed_overall = tv_elapsed (state->tv_start, tv_now);
116 rate_overall = (state->processed_files) / elapsed_overall;
118 printf ("Processed %d", state->processed_files);
120 if (state->total_files) {
121 printf (" of %d files (", state->total_files);
122 print_formatted_seconds ((state->total_files - state->processed_files) /
124 printf (" remaining). \r");
126 printf (" files (%d files/sec.) \r", (int) rate_overall);
132 /* Examine 'path' recursively as follows:
134 * o Ask the filesystem for the mtime of 'path' (path_mtime)
136 * o Ask the database for its timestamp of 'path' (path_dbtime)
138 * o If 'path_mtime' > 'path_dbtime'
140 * o For each regular file in 'path' with mtime newer than the
141 * 'path_dbtime' call add_message to add the file to the
144 * o For each sub-directory of path, recursively call into this
147 * o Tell the database to update its time of 'path' to 'path_mtime'
149 * The 'struct stat *st' must point to a structure that has already
150 * been initialized for 'path' by calling stat().
153 add_files_recursive (notmuch_database_t *notmuch,
156 add_files_state_t *state)
159 struct dirent *e, *entry = NULL;
163 time_t path_mtime, path_dbtime;
164 notmuch_status_t status, ret = NOTMUCH_STATUS_SUCCESS;
166 /* If we're told to, we bail out on encountering a read-only
167 * directory, (with this being a clear clue from the user to
168 * Notmuch that new mail won't be arriving there and we need not
170 if (state->ignore_read_only_directories &&
171 (st->st_mode & S_IWUSR) == 0)
173 state->saw_read_only_directory = TRUE;
177 path_mtime = st->st_mtime;
179 path_dbtime = notmuch_database_get_timestamp (notmuch, path);
181 dir = opendir (path);
183 fprintf (stderr, "Error opening directory %s: %s\n",
184 path, strerror (errno));
185 ret = NOTMUCH_STATUS_FILE_ERROR;
189 entry_length = offsetof (struct dirent, d_name) +
190 pathconf (path, _PC_NAME_MAX) + 1;
191 entry = malloc (entry_length);
194 err = readdir_r (dir, entry, &e);
196 fprintf (stderr, "Error reading directory: %s\n",
198 ret = NOTMUCH_STATUS_FILE_ERROR;
205 /* If this directory hasn't been modified since the last
206 * add_files, then we only need to look further for
207 * sub-directories. */
208 if (path_mtime <= path_dbtime && entry->d_type != DT_DIR)
211 /* Ignore special directories to avoid infinite recursion.
212 * Also ignore the .notmuch directory.
214 /* XXX: Eventually we'll want more sophistication to let the
215 * user specify files to be ignored. */
216 if (strcmp (entry->d_name, ".") == 0 ||
217 strcmp (entry->d_name, "..") == 0 ||
218 strcmp (entry->d_name, ".notmuch") ==0)
223 next = g_strdup_printf ("%s/%s", path, entry->d_name);
225 if (stat (next, st)) {
226 fprintf (stderr, "Error reading %s: %s\n",
227 next, strerror (errno));
228 ret = NOTMUCH_STATUS_FILE_ERROR;
232 if (S_ISREG (st->st_mode)) {
233 /* If the file hasn't been modified since the last
234 * add_files, then we need not look at it. */
235 if (st->st_mtime > path_dbtime) {
236 state->processed_files++;
238 status = notmuch_database_add_message (notmuch, next);
241 case NOTMUCH_STATUS_SUCCESS:
242 state->added_messages++;
244 /* Non-fatal issues (go on to next file) */
245 case NOTMUCH_STATUS_DUPLICATE_MESSAGE_ID:
246 /* Stay silent on this one. */
248 case NOTMUCH_STATUS_FILE_NOT_EMAIL:
249 fprintf (stderr, "Note: Ignoring non-mail file: %s\n",
252 /* Fatal issues. Don't process anymore. */
253 case NOTMUCH_STATUS_XAPIAN_EXCEPTION:
254 fprintf (stderr, "A Xapian error was encountered. Halting processing.\n");
258 fprintf (stderr, "Internal error: add_message returned unexpected value: %d\n", status);
262 if (state->processed_files % 1000 == 0)
263 add_files_print_progress (state);
265 } else if (S_ISDIR (st->st_mode)) {
266 status = add_files_recursive (notmuch, next, st, state);
267 if (status && ret == NOTMUCH_STATUS_SUCCESS)
275 status = notmuch_database_set_timestamp (notmuch, path, path_mtime);
276 if (status && ret == NOTMUCH_STATUS_SUCCESS)
290 /* This is the top-level entry point for add_files. It does a couple
291 * of error checks, and then calls into the recursive function,
292 * (avoiding the repeating of these error checks at every
293 * level---which would be useless becaues we already do a stat() at
294 * the level above). */
295 static notmuch_status_t
296 add_files (notmuch_database_t *notmuch,
298 add_files_state_t *state)
302 if (stat (path, &st)) {
303 fprintf (stderr, "Error reading directory %s: %s\n",
304 path, strerror (errno));
305 return NOTMUCH_STATUS_FILE_ERROR;
308 if (! S_ISDIR (st.st_mode)) {
309 fprintf (stderr, "Error: %s is not a directory.\n", path);
310 return NOTMUCH_STATUS_FILE_ERROR;
313 return add_files_recursive (notmuch, path, &st, state);
316 /* Recursively count all regular files in path and all sub-direcotries
317 * of path. The result is added to *count (which should be
318 * initialized to zero by the top-level caller before calling
321 count_files (const char *path, int *count)
324 struct dirent *entry, *e;
330 dir = opendir (path);
333 fprintf (stderr, "Warning: failed to open directory %s: %s\n",
334 path, strerror (errno));
338 entry_length = offsetof (struct dirent, d_name) +
339 pathconf (path, _PC_NAME_MAX) + 1;
340 entry = malloc (entry_length);
343 err = readdir_r (dir, entry, &e);
345 fprintf (stderr, "Error reading directory: %s\n",
354 /* Ignore special directories to avoid infinite recursion.
355 * Also ignore the .notmuch directory.
357 /* XXX: Eventually we'll want more sophistication to let the
358 * user specify files to be ignored. */
359 if (strcmp (entry->d_name, ".") == 0 ||
360 strcmp (entry->d_name, "..") == 0 ||
361 strcmp (entry->d_name, ".notmuch") == 0)
366 next = g_strdup_printf ("%s/%s", path, entry->d_name);
370 if (S_ISREG (st.st_mode)) {
372 if (*count % 1000 == 0) {
373 printf ("Found %d files so far.\r", *count);
376 } else if (S_ISDIR (st.st_mode)) {
377 count_files (next, count);
389 setup_command (int argc, char *argv[])
391 notmuch_database_t *notmuch = NULL;
392 char *default_path, *mail_directory = NULL;
395 add_files_state_t add_files_state;
397 struct timeval tv_now;
398 notmuch_status_t ret = NOTMUCH_STATUS_SUCCESS;
400 printf ("Welcome to notmuch!\n\n");
402 printf ("The goal of notmuch is to help you manage and search your collection of\n"
403 "email, and to efficiently keep up with the flow of email as it comes in.\n\n");
405 printf ("Notmuch needs to know the top-level directory of your email archive,\n"
406 "(where you already have mail stored and where messages will be delivered\n"
407 "in the future). This directory can contain any number of sub-directories\n"
408 "and primarily just files with indvidual email messages (eg. maildir or mh\n"
409 "archives are perfect). If there are other, non-email files (such as\n"
410 "indexes maintained by other email programs) then notmuch will do its\n"
411 "best to detect those and ignore them.\n\n");
413 printf ("Mail storage that uses mbox format, (where one mbox file contains many\n"
414 "messages), will not work with notmuch. If that's how your mail is currently\n"
415 "stored, we recommend you first convert it to maildir format with a utility\n"
416 "such as mb2md. In that case, press Control-C now and run notmuch again\n"
417 "once the conversion is complete.\n\n");
420 default_path = notmuch_database_default_path ();
421 printf ("Top-level mail directory [%s]: ", default_path);
424 getline (&mail_directory, &line_size, stdin);
425 chomp_newline (mail_directory);
429 if (mail_directory == NULL || strlen (mail_directory) == 0) {
431 free (mail_directory);
432 mail_directory = default_path;
434 /* XXX: Instead of telling the user to use an environment
435 * variable here, we should really be writing out a configuration
436 * file and loading that on the next run. */
437 if (strcmp (mail_directory, default_path)) {
438 printf ("Note: Since you are not using the default path, you will want to set\n"
439 "the NOTMUCH_BASE environment variable to %s so that\n"
440 "future calls to notmuch commands will know where to find your mail.\n",
442 printf ("For example, if you are using bash for your shell, add:\n\n");
443 printf ("\texport NOTMUCH_BASE=%s\n\n", mail_directory);
444 printf ("to your ~/.bashrc file.\n\n");
449 notmuch = notmuch_database_create (mail_directory);
450 if (notmuch == NULL) {
451 fprintf (stderr, "Failed to create new notmuch database at %s\n",
453 ret = NOTMUCH_STATUS_FILE_ERROR;
457 printf ("OK. Let's take a look at the mail we can find in the directory\n");
458 printf ("%s ...\n", mail_directory);
461 count_files (mail_directory, &count);
463 printf ("Found %d total files. That's not much mail.\n\n", count);
465 printf ("Next, we'll inspect the messages and create a database of threads:\n");
467 add_files_state.ignore_read_only_directories = FALSE;
468 add_files_state.saw_read_only_directory = FALSE;
469 add_files_state.total_files = count;
470 add_files_state.processed_files = 0;
471 add_files_state.added_messages = 0;
472 gettimeofday (&add_files_state.tv_start, NULL);
474 ret = add_files (notmuch, mail_directory, &add_files_state);
476 gettimeofday (&tv_now, NULL);
477 elapsed = tv_elapsed (add_files_state.tv_start,
479 printf ("Processed %d %s in ", add_files_state.processed_files,
480 add_files_state.processed_files == 1 ?
481 "file" : "total files");
482 print_formatted_seconds (elapsed);
484 printf (" (%d files/sec.). \n",
485 (int) (add_files_state.processed_files / elapsed));
489 if (add_files_state.added_messages) {
490 printf ("Added %d %s to the database.\n\n",
491 add_files_state.added_messages,
492 add_files_state.added_messages == 1 ?
493 "message" : "unique messages");
496 printf ("When new mail is delivered to %s in the future,\n"
497 "run \"notmuch new\" to add it to the database.\n\n",
501 printf ("Note: At least one error was encountered: %s\n",
502 notmuch_status_to_string (ret));
507 free (mail_directory);
509 notmuch_database_close (notmuch);
515 new_command (int argc, char *argv[])
517 notmuch_database_t *notmuch;
518 const char *mail_directory;
519 add_files_state_t add_files_state;
521 struct timeval tv_now;
524 notmuch = notmuch_database_open (NULL);
525 if (notmuch == NULL) {
530 mail_directory = notmuch_database_get_path (notmuch);
532 add_files_state.ignore_read_only_directories = TRUE;
533 add_files_state.saw_read_only_directory = FALSE;
534 add_files_state.total_files = 0;
535 add_files_state.processed_files = 0;
536 add_files_state.added_messages = 0;
537 gettimeofday (&add_files_state.tv_start, NULL);
539 ret = add_files (notmuch, mail_directory, &add_files_state);
541 gettimeofday (&tv_now, NULL);
542 elapsed = tv_elapsed (add_files_state.tv_start,
544 if (add_files_state.processed_files) {
545 printf ("Processed %d %s in ", add_files_state.processed_files,
546 add_files_state.processed_files == 1 ?
547 "file" : "total files");
548 print_formatted_seconds (elapsed);
550 printf (" (%d files/sec.). \n",
551 (int) (add_files_state.processed_files / elapsed));
556 if (add_files_state.added_messages) {
557 printf ("Added %d new %s to the database (not much, really).\n",
558 add_files_state.added_messages,
559 add_files_state.added_messages == 1 ?
560 "message" : "messages");
562 printf ("No new mail---and that's not much.\n");
565 if (elapsed > 1 && ! add_files_state.saw_read_only_directory) {
566 printf ("\nTip: If you have any sub-directories that are archives (that is,\n"
567 "they will never receive new mail), marking these directores as\n"
568 "read-only (chmod u-w /path/to/dir) will make \"notmuch new\"\n"
569 "much more efficient (it won't even look in those directories).\n");
573 printf ("\nNote: At least one error was encountered: %s\n",
574 notmuch_status_to_string (ret));
579 notmuch_database_close (notmuch);
585 search_command (int argc, char *argv[])
587 void *local = talloc_new (NULL);
588 notmuch_database_t *notmuch = NULL;
589 notmuch_query_t *query;
590 notmuch_results_t *results;
591 notmuch_message_t *message;
592 notmuch_tags_t *tags;
595 notmuch_status_t ret = NOTMUCH_STATUS_SUCCESS;
597 notmuch = notmuch_database_open (NULL);
598 if (notmuch == NULL) {
603 /* XXX: Should add xtalloc wrappers here and use them. */
604 query_str = talloc_strdup (local, "");
606 for (i = 0; i < argc; i++) {
608 query_str = talloc_asprintf_append (query_str, " ");
610 query_str = talloc_asprintf_append (query_str, "%s", argv[i]);
613 query = notmuch_query_create (notmuch, query_str);
615 fprintf (stderr, "Out of memory\n");
620 for (results = notmuch_query_search (query);
621 notmuch_results_has_more (results);
622 notmuch_results_advance (results))
625 message = notmuch_results_get (results);
627 printf ("%s (", notmuch_message_get_message_id (message));
629 for (tags = notmuch_message_get_tags (message);
630 notmuch_tags_has_more (tags);
631 notmuch_tags_advance (tags))
636 printf ("%s", notmuch_tags_get (tags));
643 notmuch_message_destroy (message);
646 notmuch_query_destroy (query);
650 notmuch_database_close (notmuch);
657 show_command (int argc, char *argv[])
659 fprintf (stderr, "Error: show is not implemented yet.\n");
664 dump_command (int argc, char *argv[])
667 notmuch_database_t *notmuch = NULL;
668 notmuch_query_t *query;
669 notmuch_results_t *results;
670 notmuch_message_t *message;
671 notmuch_tags_t *tags;
675 output = fopen (argv[0], "w");
676 if (output == NULL) {
677 fprintf (stderr, "Error opening %s for writing: %s\n",
678 argv[0], strerror (errno));
686 notmuch = notmuch_database_open (NULL);
687 if (notmuch == NULL) {
692 query = notmuch_query_create (notmuch, "");
694 fprintf (stderr, "Out of memory\n");
699 notmuch_query_set_sort (query, NOTMUCH_SORT_MESSAGE_ID);
701 for (results = notmuch_query_search (query);
702 notmuch_results_has_more (results);
703 notmuch_results_advance (results))
706 message = notmuch_results_get (results);
709 "%s (", notmuch_message_get_message_id (message));
711 for (tags = notmuch_message_get_tags (message);
712 notmuch_tags_has_more (tags);
713 notmuch_tags_advance (tags))
716 fprintf (output, " ");
718 fprintf (output, "%s", notmuch_tags_get (tags));
723 fprintf (output, ")\n");
725 notmuch_message_destroy (message);
728 notmuch_query_destroy (query);
732 notmuch_database_close (notmuch);
733 if (output != stdout)
740 restore_command (int argc, char *argv[])
743 notmuch_database_t *notmuch = NULL;
745 size_t line_size, line_len;
751 input = fopen (argv[0], "r");
753 fprintf (stderr, "Error opening %s for reading: %s\n",
754 argv[0], strerror (errno));
759 printf ("No filename given. Reading dump from stdin.\n");
763 notmuch = notmuch_database_open (NULL);
764 if (notmuch == NULL) {
769 /* Dump output is one line per message. We match a sequence of
770 * non-space characters for the message-id, then one or more
771 * spaces, then a list of space-separated tags as a sequence of
772 * characters within literal '(' and ')'. */
774 "^([^ ]+) \\(([^)]*)\\)$",
777 while ((line_len = getline (&line, &line_size, input)) != -1) {
779 char *message_id, *tags, *tag, *next;
780 notmuch_message_t *message;
781 notmuch_status_t status;
783 chomp_newline (line);
785 rerr = xregexec (®ex, line, 3, match, 0);
786 if (rerr == REG_NOMATCH)
788 fprintf (stderr, "Warning: Ignoring invalid input line: %s\n",
793 message_id = xstrndup (line + match[1].rm_so,
794 match[1].rm_eo - match[1].rm_so);
795 tags = xstrndup (line + match[2].rm_so,
796 match[2].rm_eo - match[2].rm_so);
800 message = notmuch_database_find_message (notmuch, message_id);
801 if (message == NULL) {
802 fprintf (stderr, "Warning: Cannot apply tags to missing message: %s (",
808 tag = strsep (&next, " ");
812 status = notmuch_message_add_tag (message, tag);
815 "Error applying tag %s to message %s:\n",
817 fprintf (stderr, "%s\n",
818 notmuch_status_to_string (status));
821 fprintf (stderr, "%s ", tag);
826 notmuch_message_destroy (message);
828 fprintf (stderr, ")\n");
840 notmuch_database_close (notmuch);
845 command_t commands[] = {
846 { "setup", setup_command,
847 "Interactively setup notmuch for first use.\n\n"
848 "\t\tInvoking notmuch with no command argument will run setup if\n"
849 "\t\tthe setup command has not previously been completed." },
850 { "new", new_command,
851 "Find and import any new messages.\n\n"
852 "\t\tScans all sub-directories of the database, adding new files\n"
853 "\t\tthat are found. Note: \"notmuch new\" will skip any\n"
854 "\t\tread-only directories, so you can use that to mark\n"
855 "\t\tdirectories that will not receive any new mail."},
856 { "search", search_command,
857 "<search-term> [...]\n\n"
858 "\t\tSearch for threads matching the given search terms.\n"
859 "\t\tOnce we actually implement search we'll document the\n"
860 "\t\tsyntax here." },
861 { "show", show_command,
863 "\t\tShow the thread with the given thread ID (see 'search')." },
864 { "dump", dump_command,
866 "\t\tCreate a plain-text dump of the tags for each message\n"
867 "\t\twriting to the given filename, if any, or to stdout.\n"
868 "\t\tThese tags are the only data in the notmuch database\n"
869 "\t\tthat can't be recreated from the messages themselves.\n"
870 "\t\tThe output of notmuch dump is therefore the only\n"
871 "\t\tcritical thing to backup (and much more friendly to\n"
872 "\t\tincremental backup than the native database files." },
873 { "restore", restore_command,
875 "\t\tRestore the tags from the given dump file (see 'dump')." }
884 fprintf (stderr, "Usage: notmuch <command> [args...]\n");
885 fprintf (stderr, "\n");
886 fprintf (stderr, "Where <command> and [args...] are as follows:\n");
887 fprintf (stderr, "\n");
889 for (i = 0; i < ARRAY_SIZE (commands); i++) {
890 command = &commands[i];
892 fprintf (stderr, "\t%s\t%s\n\n", command->name, command->usage);
897 main (int argc, char *argv[])
903 return setup_command (0, NULL);
905 for (i = 0; i < ARRAY_SIZE (commands); i++) {
906 command = &commands[i];
908 if (strcmp (argv[1], command->name) == 0)
909 return (command->function) (argc - 2, &argv[2]);
912 /* Don't complain about "help" being an unknown command when we're
913 about to provide exactly what's wanted anyway. */
914 if (strcmp (argv[1], "help") == 0 ||
915 strcmp (argv[1], "--help") == 0)
917 fprintf (stderr, "The notmuch mail system.\n\n");
919 fprintf (stderr, "Error: Unknown command '%s'\n\n", argv[1]);