1 /* notmuch - Not much of an email program, (just index and search)
3 * Copyright © 2009 Carl Worth
5 * This program is free software: you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation, either version 3 of the License, or
8 * (at your option) any later version.
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
15 * You should have received a copy of the GNU General Public License
16 * along with this program. If not, see http://www.gnu.org/licenses/ .
18 * Author: Carl Worth <cworth@cworth.org>
22 #define _GNU_SOURCE /* for getline */
28 /* This is separate from notmuch-private.h because we're trying to
29 * keep notmuch.c from looking into any internals, (which helps us
30 * develop notmuch.h into a plausible library interface).
44 #define unused(x) x __attribute__ ((unused))
46 /* There's no point in continuing when we've detected that we've done
47 * something wrong internally (as opposed to the user passing in a
50 * Note that __location__ comes from talloc.h.
52 #define INTERNAL_ERROR(format, ...) \
55 "Internal error: " format " (%s)\n", \
56 ##__VA_ARGS__, __location__); \
60 #define ARRAY_SIZE(arr) (sizeof (arr) / sizeof (arr[0]))
62 typedef int (*command_function_t) (int argc, char *argv[]);
64 typedef struct command {
66 command_function_t function;
70 typedef void (*add_files_callback_t) (notmuch_message_t *message);
73 int ignore_read_only_directories;
74 int saw_read_only_directory;
79 struct timeval tv_start;
81 add_files_callback_t callback;
85 chomp_newline (char *str)
87 if (str && str[strlen(str)-1] == '\n')
88 str[strlen(str)-1] = '\0';
91 /* Compute the number of seconds elapsed from start to end. */
93 tv_elapsed (struct timeval start, struct timeval end)
95 return ((end.tv_sec - start.tv_sec) +
96 (end.tv_usec - start.tv_usec) / 1e6);
100 print_formatted_seconds (double seconds)
106 printf ("almost no time");
110 if (seconds > 3600) {
111 hours = (int) seconds / 3600;
112 printf ("%dh ", hours);
113 seconds -= hours * 3600;
117 minutes = (int) seconds / 60;
118 printf ("%dm ", minutes);
119 seconds -= minutes * 60;
122 printf ("%ds", (int) seconds);
126 add_files_print_progress (add_files_state_t *state)
128 struct timeval tv_now;
129 double elapsed_overall, rate_overall;
131 gettimeofday (&tv_now, NULL);
133 elapsed_overall = tv_elapsed (state->tv_start, tv_now);
134 rate_overall = (state->processed_files) / elapsed_overall;
136 printf ("Processed %d", state->processed_files);
138 if (state->total_files) {
139 printf (" of %d files (", state->total_files);
140 print_formatted_seconds ((state->total_files - state->processed_files) /
142 printf (" remaining). \r");
144 printf (" files (%d files/sec.) \r", (int) rate_overall);
150 /* Examine 'path' recursively as follows:
152 * o Ask the filesystem for the mtime of 'path' (path_mtime)
154 * o Ask the database for its timestamp of 'path' (path_dbtime)
156 * o If 'path_mtime' > 'path_dbtime'
158 * o For each regular file in 'path' with mtime newer than the
159 * 'path_dbtime' call add_message to add the file to the
162 * o For each sub-directory of path, recursively call into this
165 * o Tell the database to update its time of 'path' to 'path_mtime'
167 * The 'struct stat *st' must point to a structure that has already
168 * been initialized for 'path' by calling stat().
170 static notmuch_status_t
171 add_files_recursive (notmuch_database_t *notmuch,
174 add_files_state_t *state)
177 struct dirent *e, *entry = NULL;
181 time_t path_mtime, path_dbtime;
182 notmuch_status_t status, ret = NOTMUCH_STATUS_SUCCESS;
183 notmuch_message_t *message = NULL, **closure;
185 /* If we're told to, we bail out on encountering a read-only
186 * directory, (with this being a clear clue from the user to
187 * Notmuch that new mail won't be arriving there and we need not
189 if (state->ignore_read_only_directories &&
190 (st->st_mode & S_IWUSR) == 0)
192 state->saw_read_only_directory = TRUE;
196 path_mtime = st->st_mtime;
198 path_dbtime = notmuch_database_get_timestamp (notmuch, path);
200 dir = opendir (path);
202 fprintf (stderr, "Error opening directory %s: %s\n",
203 path, strerror (errno));
204 ret = NOTMUCH_STATUS_FILE_ERROR;
208 entry_length = offsetof (struct dirent, d_name) +
209 pathconf (path, _PC_NAME_MAX) + 1;
210 entry = malloc (entry_length);
213 err = readdir_r (dir, entry, &e);
215 fprintf (stderr, "Error reading directory: %s\n",
217 ret = NOTMUCH_STATUS_FILE_ERROR;
224 /* If this directory hasn't been modified since the last
225 * add_files, then we only need to look further for
226 * sub-directories. */
227 if (path_mtime <= path_dbtime && entry->d_type != DT_DIR)
230 /* Ignore special directories to avoid infinite recursion.
231 * Also ignore the .notmuch directory.
233 /* XXX: Eventually we'll want more sophistication to let the
234 * user specify files to be ignored. */
235 if (strcmp (entry->d_name, ".") == 0 ||
236 strcmp (entry->d_name, "..") == 0 ||
237 strcmp (entry->d_name, ".notmuch") ==0)
242 next = talloc_asprintf (notmuch, "%s/%s", path, entry->d_name);
244 if (stat (next, st)) {
245 fprintf (stderr, "Error reading %s: %s\n",
246 next, strerror (errno));
247 ret = NOTMUCH_STATUS_FILE_ERROR;
251 if (S_ISREG (st->st_mode)) {
252 /* If the file hasn't been modified since the last
253 * add_files, then we need not look at it. */
254 if (st->st_mtime > path_dbtime) {
255 state->processed_files++;
262 status = notmuch_database_add_message (notmuch, next, closure);
265 case NOTMUCH_STATUS_SUCCESS:
266 state->added_messages++;
268 (state->callback) (message);
270 /* Non-fatal issues (go on to next file) */
271 case NOTMUCH_STATUS_DUPLICATE_MESSAGE_ID:
272 /* Stay silent on this one. */
274 case NOTMUCH_STATUS_FILE_NOT_EMAIL:
275 fprintf (stderr, "Note: Ignoring non-mail file: %s\n",
278 /* Fatal issues. Don't process anymore. */
279 case NOTMUCH_STATUS_XAPIAN_EXCEPTION:
280 case NOTMUCH_STATUS_OUT_OF_MEMORY:
281 fprintf (stderr, "Error: %s. Halting processing.\n",
282 notmuch_status_to_string (status));
286 case NOTMUCH_STATUS_FILE_ERROR:
287 case NOTMUCH_STATUS_NULL_POINTER:
288 case NOTMUCH_STATUS_TAG_TOO_LONG:
289 case NOTMUCH_STATUS_UNBALANCED_FREEZE_THAW:
290 case NOTMUCH_STATUS_LAST_STATUS:
291 INTERNAL_ERROR ("add_message returned unexpected value: %d", status);
296 notmuch_message_destroy (message);
300 if (state->processed_files % 1000 == 0)
301 add_files_print_progress (state);
303 } else if (S_ISDIR (st->st_mode)) {
304 status = add_files_recursive (notmuch, next, st, state);
305 if (status && ret == NOTMUCH_STATUS_SUCCESS)
313 status = notmuch_database_set_timestamp (notmuch, path, path_mtime);
314 if (status && ret == NOTMUCH_STATUS_SUCCESS)
328 /* This is the top-level entry point for add_files. It does a couple
329 * of error checks, and then calls into the recursive function,
330 * (avoiding the repeating of these error checks at every
331 * level---which would be useless becaues we already do a stat() at
332 * the level above). */
333 static notmuch_status_t
334 add_files (notmuch_database_t *notmuch,
336 add_files_state_t *state)
340 if (stat (path, &st)) {
341 fprintf (stderr, "Error reading directory %s: %s\n",
342 path, strerror (errno));
343 return NOTMUCH_STATUS_FILE_ERROR;
346 if (! S_ISDIR (st.st_mode)) {
347 fprintf (stderr, "Error: %s is not a directory.\n", path);
348 return NOTMUCH_STATUS_FILE_ERROR;
351 return add_files_recursive (notmuch, path, &st, state);
354 /* Recursively count all regular files in path and all sub-direcotries
355 * of path. The result is added to *count (which should be
356 * initialized to zero by the top-level caller before calling
359 count_files (const char *path, int *count)
362 struct dirent *e, *entry = NULL;
368 dir = opendir (path);
371 fprintf (stderr, "Warning: failed to open directory %s: %s\n",
372 path, strerror (errno));
376 entry_length = offsetof (struct dirent, d_name) +
377 pathconf (path, _PC_NAME_MAX) + 1;
378 entry = malloc (entry_length);
381 err = readdir_r (dir, entry, &e);
383 fprintf (stderr, "Error reading directory: %s\n",
392 /* Ignore special directories to avoid infinite recursion.
393 * Also ignore the .notmuch directory.
395 /* XXX: Eventually we'll want more sophistication to let the
396 * user specify files to be ignored. */
397 if (strcmp (entry->d_name, ".") == 0 ||
398 strcmp (entry->d_name, "..") == 0 ||
399 strcmp (entry->d_name, ".notmuch") == 0)
404 if (asprintf (&next, "%s/%s", path, entry->d_name) == -1) {
406 fprintf (stderr, "Error descending from %s to %s: Out of memory\n",
407 path, entry->d_name);
413 if (S_ISREG (st.st_mode)) {
415 if (*count % 1000 == 0) {
416 printf ("Found %d files so far.\r", *count);
419 } else if (S_ISDIR (st.st_mode)) {
420 count_files (next, count);
434 setup_command (unused (int argc), unused (char *argv[]))
436 notmuch_database_t *notmuch = NULL;
437 char *default_path, *mail_directory = NULL;
440 add_files_state_t add_files_state;
442 struct timeval tv_now;
443 notmuch_status_t ret = NOTMUCH_STATUS_SUCCESS;
445 printf ("Welcome to notmuch!\n\n");
447 printf ("The goal of notmuch is to help you manage and search your collection of\n"
448 "email, and to efficiently keep up with the flow of email as it comes in.\n\n");
450 printf ("Notmuch needs to know the top-level directory of your email archive,\n"
451 "(where you already have mail stored and where messages will be delivered\n"
452 "in the future). This directory can contain any number of sub-directories\n"
453 "and primarily just files with indvidual email messages (eg. maildir or mh\n"
454 "archives are perfect). If there are other, non-email files (such as\n"
455 "indexes maintained by other email programs) then notmuch will do its\n"
456 "best to detect those and ignore them.\n\n");
458 printf ("Mail storage that uses mbox format, (where one mbox file contains many\n"
459 "messages), will not work with notmuch. If that's how your mail is currently\n"
460 "stored, we recommend you first convert it to maildir format with a utility\n"
461 "such as mb2md. In that case, press Control-C now and run notmuch again\n"
462 "once the conversion is complete.\n\n");
465 default_path = notmuch_database_default_path ();
466 printf ("Top-level mail directory [%s]: ", default_path);
469 getline (&mail_directory, &line_size, stdin);
470 chomp_newline (mail_directory);
474 if (mail_directory == NULL || strlen (mail_directory) == 0) {
476 free (mail_directory);
477 mail_directory = default_path;
479 /* XXX: Instead of telling the user to use an environment
480 * variable here, we should really be writing out a configuration
481 * file and loading that on the next run. */
482 if (strcmp (mail_directory, default_path)) {
483 printf ("Note: Since you are not using the default path, you will want to set\n"
484 "the NOTMUCH_BASE environment variable to %s so that\n"
485 "future calls to notmuch commands will know where to find your mail.\n",
487 printf ("For example, if you are using bash for your shell, add:\n\n");
488 printf ("\texport NOTMUCH_BASE=%s\n\n", mail_directory);
489 printf ("to your ~/.bashrc file.\n\n");
494 notmuch = notmuch_database_create (mail_directory);
495 if (notmuch == NULL) {
496 fprintf (stderr, "Failed to create new notmuch database at %s\n",
498 ret = NOTMUCH_STATUS_FILE_ERROR;
502 printf ("OK. Let's take a look at the mail we can find in the directory\n");
503 printf ("%s ...\n", mail_directory);
506 count_files (mail_directory, &count);
508 printf ("Found %d total files. That's not much mail.\n\n", count);
510 printf ("Next, we'll inspect the messages and create a database of threads:\n");
512 add_files_state.ignore_read_only_directories = FALSE;
513 add_files_state.saw_read_only_directory = FALSE;
514 add_files_state.total_files = count;
515 add_files_state.processed_files = 0;
516 add_files_state.added_messages = 0;
517 add_files_state.callback = NULL;
518 gettimeofday (&add_files_state.tv_start, NULL);
520 ret = add_files (notmuch, mail_directory, &add_files_state);
522 gettimeofday (&tv_now, NULL);
523 elapsed = tv_elapsed (add_files_state.tv_start,
525 printf ("Processed %d %s in ", add_files_state.processed_files,
526 add_files_state.processed_files == 1 ?
527 "file" : "total files");
528 print_formatted_seconds (elapsed);
530 printf (" (%d files/sec.). \n",
531 (int) (add_files_state.processed_files / elapsed));
535 if (add_files_state.added_messages) {
536 printf ("Added %d %s to the database.\n\n",
537 add_files_state.added_messages,
538 add_files_state.added_messages == 1 ?
539 "message" : "unique messages");
542 printf ("When new mail is delivered to %s in the future,\n"
543 "run \"notmuch new\" to add it to the database.\n\n",
547 printf ("Note: At least one error was encountered: %s\n",
548 notmuch_status_to_string (ret));
553 free (mail_directory);
555 notmuch_database_close (notmuch);
561 tag_inbox_and_unread (notmuch_message_t *message)
563 notmuch_message_add_tag (message, "inbox");
564 notmuch_message_add_tag (message, "unread");
568 new_command (unused (int argc), unused (char *argv[]))
570 notmuch_database_t *notmuch;
571 const char *mail_directory;
572 add_files_state_t add_files_state;
574 struct timeval tv_now;
577 notmuch = notmuch_database_open (NULL);
578 if (notmuch == NULL) {
583 mail_directory = notmuch_database_get_path (notmuch);
585 add_files_state.ignore_read_only_directories = TRUE;
586 add_files_state.saw_read_only_directory = FALSE;
587 add_files_state.total_files = 0;
588 add_files_state.processed_files = 0;
589 add_files_state.added_messages = 0;
590 add_files_state.callback = tag_inbox_and_unread;
591 gettimeofday (&add_files_state.tv_start, NULL);
593 ret = add_files (notmuch, mail_directory, &add_files_state);
595 gettimeofday (&tv_now, NULL);
596 elapsed = tv_elapsed (add_files_state.tv_start,
598 if (add_files_state.processed_files) {
599 printf ("Processed %d %s in ", add_files_state.processed_files,
600 add_files_state.processed_files == 1 ?
601 "file" : "total files");
602 print_formatted_seconds (elapsed);
604 printf (" (%d files/sec.). \n",
605 (int) (add_files_state.processed_files / elapsed));
610 if (add_files_state.added_messages) {
611 printf ("Added %d new %s to the database (not much, really).\n",
612 add_files_state.added_messages,
613 add_files_state.added_messages == 1 ?
614 "message" : "messages");
616 printf ("No new mail---and that's not much.\n");
619 if (elapsed > 1 && ! add_files_state.saw_read_only_directory) {
620 printf ("\nTip: If you have any sub-directories that are archives (that is,\n"
621 "they will never receive new mail), marking these directores as\n"
622 "read-only (chmod u-w /path/to/dir) will make \"notmuch new\"\n"
623 "much more efficient (it won't even look in those directories).\n");
627 printf ("\nNote: At least one error was encountered: %s\n",
628 notmuch_status_to_string (ret));
633 notmuch_database_close (notmuch);
639 search_command (int argc, char *argv[])
641 void *local = talloc_new (NULL);
642 notmuch_database_t *notmuch = NULL;
643 notmuch_query_t *query;
644 notmuch_thread_results_t *results;
645 notmuch_thread_t *thread;
646 notmuch_tags_t *tags;
649 notmuch_status_t ret = NOTMUCH_STATUS_SUCCESS;
651 notmuch = notmuch_database_open (NULL);
652 if (notmuch == NULL) {
657 /* XXX: Should add xtalloc wrappers here and use them. */
658 query_str = talloc_strdup (local, "");
660 for (i = 0; i < argc; i++) {
662 query_str = talloc_asprintf_append (query_str, " ");
664 query_str = talloc_asprintf_append (query_str, "%s", argv[i]);
667 query = notmuch_query_create (notmuch, query_str);
669 fprintf (stderr, "Out of memory\n");
674 for (results = notmuch_query_search_threads (query);
675 notmuch_thread_results_has_more (results);
676 notmuch_thread_results_advance (results))
680 thread = notmuch_thread_results_get (results);
683 notmuch_thread_get_thread_id (thread),
684 notmuch_thread_get_subject (thread));
687 for (tags = notmuch_thread_get_tags (thread);
688 notmuch_tags_has_more (tags);
689 notmuch_tags_advance (tags))
693 printf ("%s", notmuch_tags_get (tags));
698 notmuch_thread_destroy (thread);
701 notmuch_query_destroy (query);
705 notmuch_database_close (notmuch);
712 show_command (unused (int argc), unused (char *argv[]))
714 fprintf (stderr, "Error: show is not implemented yet.\n");
719 dump_command (int argc, char *argv[])
722 notmuch_database_t *notmuch = NULL;
723 notmuch_query_t *query;
724 notmuch_message_results_t *results;
725 notmuch_message_t *message;
726 notmuch_tags_t *tags;
730 output = fopen (argv[0], "w");
731 if (output == NULL) {
732 fprintf (stderr, "Error opening %s for writing: %s\n",
733 argv[0], strerror (errno));
741 notmuch = notmuch_database_open (NULL);
742 if (notmuch == NULL) {
747 query = notmuch_query_create (notmuch, "");
749 fprintf (stderr, "Out of memory\n");
754 notmuch_query_set_sort (query, NOTMUCH_SORT_MESSAGE_ID);
756 for (results = notmuch_query_search_messages (query);
757 notmuch_message_results_has_more (results);
758 notmuch_message_results_advance (results))
761 message = notmuch_message_results_get (results);
764 "%s (", notmuch_message_get_message_id (message));
766 for (tags = notmuch_message_get_tags (message);
767 notmuch_tags_has_more (tags);
768 notmuch_tags_advance (tags))
771 fprintf (output, " ");
773 fprintf (output, "%s", notmuch_tags_get (tags));
778 fprintf (output, ")\n");
780 notmuch_message_destroy (message);
783 notmuch_query_destroy (query);
787 notmuch_database_close (notmuch);
788 if (output && output != stdout)
795 restore_command (int argc, char *argv[])
798 notmuch_database_t *notmuch = NULL;
807 input = fopen (argv[0], "r");
809 fprintf (stderr, "Error opening %s for reading: %s\n",
810 argv[0], strerror (errno));
815 printf ("No filename given. Reading dump from stdin.\n");
819 notmuch = notmuch_database_open (NULL);
820 if (notmuch == NULL) {
825 /* Dump output is one line per message. We match a sequence of
826 * non-space characters for the message-id, then one or more
827 * spaces, then a list of space-separated tags as a sequence of
828 * characters within literal '(' and ')'. */
830 "^([^ ]+) \\(([^)]*)\\)$",
833 while ((line_len = getline (&line, &line_size, input)) != -1) {
835 char *message_id, *tags, *tag, *next;
836 notmuch_message_t *message;
837 notmuch_status_t status;
839 chomp_newline (line);
841 rerr = xregexec (®ex, line, 3, match, 0);
842 if (rerr == REG_NOMATCH)
844 fprintf (stderr, "Warning: Ignoring invalid input line: %s\n",
849 message_id = xstrndup (line + match[1].rm_so,
850 match[1].rm_eo - match[1].rm_so);
851 tags = xstrndup (line + match[2].rm_so,
852 match[2].rm_eo - match[2].rm_so);
856 message = notmuch_database_find_message (notmuch, message_id);
857 if (message == NULL) {
858 fprintf (stderr, "Warning: Cannot apply tags to missing message: %s\n",
863 notmuch_message_freeze (message);
865 notmuch_message_remove_all_tags (message);
869 tag = strsep (&next, " ");
872 status = notmuch_message_add_tag (message, tag);
875 "Error applying tag %s to message %s:\n",
877 fprintf (stderr, "%s\n",
878 notmuch_status_to_string (status));
882 notmuch_message_thaw (message);
883 notmuch_message_destroy (message);
896 notmuch_database_close (notmuch);
897 if (input && input != stdin)
903 command_t commands[] = {
904 { "setup", setup_command,
905 "Interactively setup notmuch for first use.\n\n"
906 "\t\tInvoking notmuch with no command argument will run setup if\n"
907 "\t\tthe setup command has not previously been completed." },
908 { "new", new_command,
909 "Find and import any new messages.\n\n"
910 "\t\tScans all sub-directories of the database, adding new messages\n"
911 "\t\tthat are found. Each new message will be tagges as both\n"
912 "\t\t\"inbox\" and \"unread\".\n"
914 "\t\tNote: \"notmuch new\" will skip any read-only directories,\n"
915 "\t\tso you can use that to mark tdirectories that will not\n"
916 "\t\treceive any new mail (and make \"notmuch new\" faster)." },
917 { "search", search_command,
918 "<search-term> [...]\n\n"
919 "\t\tSearch for threads matching the given search terms.\n"
920 "\t\tOnce we actually implement search we'll document the\n"
921 "\t\tsyntax here." },
922 { "show", show_command,
924 "\t\tShow the thread with the given thread ID (see 'search')." },
925 { "dump", dump_command,
927 "\t\tCreate a plain-text dump of the tags for each message\n"
928 "\t\twriting to the given filename, if any, or to stdout.\n"
929 "\t\tThese tags are the only data in the notmuch database\n"
930 "\t\tthat can't be recreated from the messages themselves.\n"
931 "\t\tThe output of notmuch dump is therefore the only\n"
932 "\t\tcritical thing to backup (and much more friendly to\n"
933 "\t\tincremental backup than the native database files." },
934 { "restore", restore_command,
936 "\t\tRestore the tags from the given dump file (see 'dump')." }
945 fprintf (stderr, "Usage: notmuch <command> [args...]\n");
946 fprintf (stderr, "\n");
947 fprintf (stderr, "Where <command> and [args...] are as follows:\n");
948 fprintf (stderr, "\n");
950 for (i = 0; i < ARRAY_SIZE (commands); i++) {
951 command = &commands[i];
953 fprintf (stderr, "\t%s\t%s\n\n", command->name, command->usage);
958 main (int argc, char *argv[])
964 return setup_command (0, NULL);
966 for (i = 0; i < ARRAY_SIZE (commands); i++) {
967 command = &commands[i];
969 if (strcmp (argv[1], command->name) == 0)
970 return (command->function) (argc - 2, &argv[2]);
973 /* Don't complain about "help" being an unknown command when we're
974 about to provide exactly what's wanted anyway. */
975 if (strcmp (argv[1], "help") == 0 ||
976 strcmp (argv[1], "--help") == 0)
978 fprintf (stderr, "The notmuch mail system.\n\n");
982 fprintf (stderr, "Error: Unknown command '%s'\n\n", argv[1]);