1 /* notmuch - Not much of an email program, (just index and search)
3 * Copyright © 2009 Carl Worth
5 * This program is free software: you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation, either version 3 of the License, or
8 * (at your option) any later version.
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
15 * You should have received a copy of the GNU General Public License
16 * along with this program. If not, see http://www.gnu.org/licenses/ .
18 * Author: Carl Worth <cworth@cworth.org>
22 #define _GNU_SOURCE /* for getline */
28 /* This is separate from notmuch-private.h because we're trying to
29 * keep notmuch.c from looking into any internals, (which helps us
30 * develop notmuch.h into a plausible library interface).
44 #include <glib.h> /* g_strdup_printf */
46 #define unused(x) x __attribute__ ((unused))
48 /* There's no point in continuing when we've detected that we've done
49 * something wrong internally (as opposed to the user passing in a
52 * Note that __location__ comes from talloc.h.
54 #define INTERNAL_ERROR(format, ...) \
57 "Internal error: " format " (%s)\n", \
58 ##__VA_ARGS__, __location__); \
62 #define ARRAY_SIZE(arr) (sizeof (arr) / sizeof (arr[0]))
64 typedef int (*command_function_t) (int argc, char *argv[]);
66 typedef struct command {
68 command_function_t function;
73 int ignore_read_only_directories;
74 int saw_read_only_directory;
79 struct timeval tv_start;
83 chomp_newline (char *str)
85 if (str && str[strlen(str)-1] == '\n')
86 str[strlen(str)-1] = '\0';
89 /* Compute the number of seconds elapsed from start to end. */
91 tv_elapsed (struct timeval start, struct timeval end)
93 return ((end.tv_sec - start.tv_sec) +
94 (end.tv_usec - start.tv_usec) / 1e6);
98 print_formatted_seconds (double seconds)
104 printf ("almost no time");
108 if (seconds > 3600) {
109 hours = (int) seconds / 3600;
110 printf ("%dh ", hours);
111 seconds -= hours * 3600;
115 minutes = (int) seconds / 60;
116 printf ("%dm ", minutes);
117 seconds -= minutes * 60;
120 printf ("%ds", (int) seconds);
124 add_files_print_progress (add_files_state_t *state)
126 struct timeval tv_now;
127 double elapsed_overall, rate_overall;
129 gettimeofday (&tv_now, NULL);
131 elapsed_overall = tv_elapsed (state->tv_start, tv_now);
132 rate_overall = (state->processed_files) / elapsed_overall;
134 printf ("Processed %d", state->processed_files);
136 if (state->total_files) {
137 printf (" of %d files (", state->total_files);
138 print_formatted_seconds ((state->total_files - state->processed_files) /
140 printf (" remaining). \r");
142 printf (" files (%d files/sec.) \r", (int) rate_overall);
148 /* Examine 'path' recursively as follows:
150 * o Ask the filesystem for the mtime of 'path' (path_mtime)
152 * o Ask the database for its timestamp of 'path' (path_dbtime)
154 * o If 'path_mtime' > 'path_dbtime'
156 * o For each regular file in 'path' with mtime newer than the
157 * 'path_dbtime' call add_message to add the file to the
160 * o For each sub-directory of path, recursively call into this
163 * o Tell the database to update its time of 'path' to 'path_mtime'
165 * The 'struct stat *st' must point to a structure that has already
166 * been initialized for 'path' by calling stat().
168 static notmuch_status_t
169 add_files_recursive (notmuch_database_t *notmuch,
172 add_files_state_t *state)
175 struct dirent *e, *entry = NULL;
179 time_t path_mtime, path_dbtime;
180 notmuch_status_t status, ret = NOTMUCH_STATUS_SUCCESS;
182 /* If we're told to, we bail out on encountering a read-only
183 * directory, (with this being a clear clue from the user to
184 * Notmuch that new mail won't be arriving there and we need not
186 if (state->ignore_read_only_directories &&
187 (st->st_mode & S_IWUSR) == 0)
189 state->saw_read_only_directory = TRUE;
193 path_mtime = st->st_mtime;
195 path_dbtime = notmuch_database_get_timestamp (notmuch, path);
197 dir = opendir (path);
199 fprintf (stderr, "Error opening directory %s: %s\n",
200 path, strerror (errno));
201 ret = NOTMUCH_STATUS_FILE_ERROR;
205 entry_length = offsetof (struct dirent, d_name) +
206 pathconf (path, _PC_NAME_MAX) + 1;
207 entry = malloc (entry_length);
210 err = readdir_r (dir, entry, &e);
212 fprintf (stderr, "Error reading directory: %s\n",
214 ret = NOTMUCH_STATUS_FILE_ERROR;
221 /* If this directory hasn't been modified since the last
222 * add_files, then we only need to look further for
223 * sub-directories. */
224 if (path_mtime <= path_dbtime && entry->d_type != DT_DIR)
227 /* Ignore special directories to avoid infinite recursion.
228 * Also ignore the .notmuch directory.
230 /* XXX: Eventually we'll want more sophistication to let the
231 * user specify files to be ignored. */
232 if (strcmp (entry->d_name, ".") == 0 ||
233 strcmp (entry->d_name, "..") == 0 ||
234 strcmp (entry->d_name, ".notmuch") ==0)
239 next = g_strdup_printf ("%s/%s", path, entry->d_name);
241 if (stat (next, st)) {
242 fprintf (stderr, "Error reading %s: %s\n",
243 next, strerror (errno));
244 ret = NOTMUCH_STATUS_FILE_ERROR;
248 if (S_ISREG (st->st_mode)) {
249 /* If the file hasn't been modified since the last
250 * add_files, then we need not look at it. */
251 if (st->st_mtime > path_dbtime) {
252 state->processed_files++;
254 status = notmuch_database_add_message (notmuch, next);
257 case NOTMUCH_STATUS_SUCCESS:
258 state->added_messages++;
260 /* Non-fatal issues (go on to next file) */
261 case NOTMUCH_STATUS_DUPLICATE_MESSAGE_ID:
262 /* Stay silent on this one. */
264 case NOTMUCH_STATUS_FILE_NOT_EMAIL:
265 fprintf (stderr, "Note: Ignoring non-mail file: %s\n",
268 /* Fatal issues. Don't process anymore. */
269 case NOTMUCH_STATUS_XAPIAN_EXCEPTION:
270 case NOTMUCH_STATUS_OUT_OF_MEMORY:
271 fprintf (stderr, "Error: %s. Halting processing.\n",
272 notmuch_status_to_string (status));
276 case NOTMUCH_STATUS_FILE_ERROR:
277 case NOTMUCH_STATUS_NULL_POINTER:
278 case NOTMUCH_STATUS_TAG_TOO_LONG:
279 case NOTMUCH_STATUS_LAST_STATUS:
280 INTERNAL_ERROR ("add_message returned unexpected value: %d", status);
283 if (state->processed_files % 1000 == 0)
284 add_files_print_progress (state);
286 } else if (S_ISDIR (st->st_mode)) {
287 status = add_files_recursive (notmuch, next, st, state);
288 if (status && ret == NOTMUCH_STATUS_SUCCESS)
296 status = notmuch_database_set_timestamp (notmuch, path, path_mtime);
297 if (status && ret == NOTMUCH_STATUS_SUCCESS)
311 /* This is the top-level entry point for add_files. It does a couple
312 * of error checks, and then calls into the recursive function,
313 * (avoiding the repeating of these error checks at every
314 * level---which would be useless becaues we already do a stat() at
315 * the level above). */
316 static notmuch_status_t
317 add_files (notmuch_database_t *notmuch,
319 add_files_state_t *state)
323 if (stat (path, &st)) {
324 fprintf (stderr, "Error reading directory %s: %s\n",
325 path, strerror (errno));
326 return NOTMUCH_STATUS_FILE_ERROR;
329 if (! S_ISDIR (st.st_mode)) {
330 fprintf (stderr, "Error: %s is not a directory.\n", path);
331 return NOTMUCH_STATUS_FILE_ERROR;
334 return add_files_recursive (notmuch, path, &st, state);
337 /* Recursively count all regular files in path and all sub-direcotries
338 * of path. The result is added to *count (which should be
339 * initialized to zero by the top-level caller before calling
342 count_files (const char *path, int *count)
345 struct dirent *entry, *e;
351 dir = opendir (path);
354 fprintf (stderr, "Warning: failed to open directory %s: %s\n",
355 path, strerror (errno));
359 entry_length = offsetof (struct dirent, d_name) +
360 pathconf (path, _PC_NAME_MAX) + 1;
361 entry = malloc (entry_length);
364 err = readdir_r (dir, entry, &e);
366 fprintf (stderr, "Error reading directory: %s\n",
375 /* Ignore special directories to avoid infinite recursion.
376 * Also ignore the .notmuch directory.
378 /* XXX: Eventually we'll want more sophistication to let the
379 * user specify files to be ignored. */
380 if (strcmp (entry->d_name, ".") == 0 ||
381 strcmp (entry->d_name, "..") == 0 ||
382 strcmp (entry->d_name, ".notmuch") == 0)
387 next = g_strdup_printf ("%s/%s", path, entry->d_name);
391 if (S_ISREG (st.st_mode)) {
393 if (*count % 1000 == 0) {
394 printf ("Found %d files so far.\r", *count);
397 } else if (S_ISDIR (st.st_mode)) {
398 count_files (next, count);
410 setup_command (unused (int argc), unused (char *argv[]))
412 notmuch_database_t *notmuch = NULL;
413 char *default_path, *mail_directory = NULL;
416 add_files_state_t add_files_state;
418 struct timeval tv_now;
419 notmuch_status_t ret = NOTMUCH_STATUS_SUCCESS;
421 printf ("Welcome to notmuch!\n\n");
423 printf ("The goal of notmuch is to help you manage and search your collection of\n"
424 "email, and to efficiently keep up with the flow of email as it comes in.\n\n");
426 printf ("Notmuch needs to know the top-level directory of your email archive,\n"
427 "(where you already have mail stored and where messages will be delivered\n"
428 "in the future). This directory can contain any number of sub-directories\n"
429 "and primarily just files with indvidual email messages (eg. maildir or mh\n"
430 "archives are perfect). If there are other, non-email files (such as\n"
431 "indexes maintained by other email programs) then notmuch will do its\n"
432 "best to detect those and ignore them.\n\n");
434 printf ("Mail storage that uses mbox format, (where one mbox file contains many\n"
435 "messages), will not work with notmuch. If that's how your mail is currently\n"
436 "stored, we recommend you first convert it to maildir format with a utility\n"
437 "such as mb2md. In that case, press Control-C now and run notmuch again\n"
438 "once the conversion is complete.\n\n");
441 default_path = notmuch_database_default_path ();
442 printf ("Top-level mail directory [%s]: ", default_path);
445 getline (&mail_directory, &line_size, stdin);
446 chomp_newline (mail_directory);
450 if (mail_directory == NULL || strlen (mail_directory) == 0) {
452 free (mail_directory);
453 mail_directory = default_path;
455 /* XXX: Instead of telling the user to use an environment
456 * variable here, we should really be writing out a configuration
457 * file and loading that on the next run. */
458 if (strcmp (mail_directory, default_path)) {
459 printf ("Note: Since you are not using the default path, you will want to set\n"
460 "the NOTMUCH_BASE environment variable to %s so that\n"
461 "future calls to notmuch commands will know where to find your mail.\n",
463 printf ("For example, if you are using bash for your shell, add:\n\n");
464 printf ("\texport NOTMUCH_BASE=%s\n\n", mail_directory);
465 printf ("to your ~/.bashrc file.\n\n");
470 notmuch = notmuch_database_create (mail_directory);
471 if (notmuch == NULL) {
472 fprintf (stderr, "Failed to create new notmuch database at %s\n",
474 ret = NOTMUCH_STATUS_FILE_ERROR;
478 printf ("OK. Let's take a look at the mail we can find in the directory\n");
479 printf ("%s ...\n", mail_directory);
482 count_files (mail_directory, &count);
484 printf ("Found %d total files. That's not much mail.\n\n", count);
486 printf ("Next, we'll inspect the messages and create a database of threads:\n");
488 add_files_state.ignore_read_only_directories = FALSE;
489 add_files_state.saw_read_only_directory = FALSE;
490 add_files_state.total_files = count;
491 add_files_state.processed_files = 0;
492 add_files_state.added_messages = 0;
493 gettimeofday (&add_files_state.tv_start, NULL);
495 ret = add_files (notmuch, mail_directory, &add_files_state);
497 gettimeofday (&tv_now, NULL);
498 elapsed = tv_elapsed (add_files_state.tv_start,
500 printf ("Processed %d %s in ", add_files_state.processed_files,
501 add_files_state.processed_files == 1 ?
502 "file" : "total files");
503 print_formatted_seconds (elapsed);
505 printf (" (%d files/sec.). \n",
506 (int) (add_files_state.processed_files / elapsed));
510 if (add_files_state.added_messages) {
511 printf ("Added %d %s to the database.\n\n",
512 add_files_state.added_messages,
513 add_files_state.added_messages == 1 ?
514 "message" : "unique messages");
517 printf ("When new mail is delivered to %s in the future,\n"
518 "run \"notmuch new\" to add it to the database.\n\n",
522 printf ("Note: At least one error was encountered: %s\n",
523 notmuch_status_to_string (ret));
528 free (mail_directory);
530 notmuch_database_close (notmuch);
536 new_command (unused (int argc), unused (char *argv[]))
538 notmuch_database_t *notmuch;
539 const char *mail_directory;
540 add_files_state_t add_files_state;
542 struct timeval tv_now;
545 notmuch = notmuch_database_open (NULL);
546 if (notmuch == NULL) {
551 mail_directory = notmuch_database_get_path (notmuch);
553 add_files_state.ignore_read_only_directories = TRUE;
554 add_files_state.saw_read_only_directory = FALSE;
555 add_files_state.total_files = 0;
556 add_files_state.processed_files = 0;
557 add_files_state.added_messages = 0;
558 gettimeofday (&add_files_state.tv_start, NULL);
560 ret = add_files (notmuch, mail_directory, &add_files_state);
562 gettimeofday (&tv_now, NULL);
563 elapsed = tv_elapsed (add_files_state.tv_start,
565 if (add_files_state.processed_files) {
566 printf ("Processed %d %s in ", add_files_state.processed_files,
567 add_files_state.processed_files == 1 ?
568 "file" : "total files");
569 print_formatted_seconds (elapsed);
571 printf (" (%d files/sec.). \n",
572 (int) (add_files_state.processed_files / elapsed));
577 if (add_files_state.added_messages) {
578 printf ("Added %d new %s to the database (not much, really).\n",
579 add_files_state.added_messages,
580 add_files_state.added_messages == 1 ?
581 "message" : "messages");
583 printf ("No new mail---and that's not much.\n");
586 if (elapsed > 1 && ! add_files_state.saw_read_only_directory) {
587 printf ("\nTip: If you have any sub-directories that are archives (that is,\n"
588 "they will never receive new mail), marking these directores as\n"
589 "read-only (chmod u-w /path/to/dir) will make \"notmuch new\"\n"
590 "much more efficient (it won't even look in those directories).\n");
594 printf ("\nNote: At least one error was encountered: %s\n",
595 notmuch_status_to_string (ret));
600 notmuch_database_close (notmuch);
606 search_command (int argc, char *argv[])
608 void *local = talloc_new (NULL);
609 notmuch_database_t *notmuch = NULL;
610 notmuch_query_t *query;
611 notmuch_thread_results_t *results;
612 notmuch_thread_t *thread;
615 notmuch_status_t ret = NOTMUCH_STATUS_SUCCESS;
617 notmuch = notmuch_database_open (NULL);
618 if (notmuch == NULL) {
623 /* XXX: Should add xtalloc wrappers here and use them. */
624 query_str = talloc_strdup (local, "");
626 for (i = 0; i < argc; i++) {
628 query_str = talloc_asprintf_append (query_str, " ");
630 query_str = talloc_asprintf_append (query_str, "%s", argv[i]);
633 query = notmuch_query_create (notmuch, query_str);
635 fprintf (stderr, "Out of memory\n");
640 for (results = notmuch_query_search_threads (query);
641 notmuch_thread_results_has_more (results);
642 notmuch_thread_results_advance (results))
644 thread = notmuch_thread_results_get (results);
646 printf ("%s\n", notmuch_thread_get_thread_id (thread));
648 notmuch_thread_destroy (thread);
651 notmuch_query_destroy (query);
655 notmuch_database_close (notmuch);
662 show_command (unused (int argc), unused (char *argv[]))
664 fprintf (stderr, "Error: show is not implemented yet.\n");
669 dump_command (int argc, char *argv[])
672 notmuch_database_t *notmuch = NULL;
673 notmuch_query_t *query;
674 notmuch_message_results_t *results;
675 notmuch_message_t *message;
676 notmuch_tags_t *tags;
680 output = fopen (argv[0], "w");
681 if (output == NULL) {
682 fprintf (stderr, "Error opening %s for writing: %s\n",
683 argv[0], strerror (errno));
691 notmuch = notmuch_database_open (NULL);
692 if (notmuch == NULL) {
697 query = notmuch_query_create (notmuch, "");
699 fprintf (stderr, "Out of memory\n");
704 notmuch_query_set_sort (query, NOTMUCH_SORT_MESSAGE_ID);
706 for (results = notmuch_query_search_messages (query);
707 notmuch_message_results_has_more (results);
708 notmuch_message_results_advance (results))
711 message = notmuch_message_results_get (results);
714 "%s (", notmuch_message_get_message_id (message));
716 for (tags = notmuch_message_get_tags (message);
717 notmuch_tags_has_more (tags);
718 notmuch_tags_advance (tags))
721 fprintf (output, " ");
723 fprintf (output, "%s", notmuch_tags_get (tags));
728 fprintf (output, ")\n");
730 notmuch_message_destroy (message);
733 notmuch_query_destroy (query);
737 notmuch_database_close (notmuch);
738 if (output && output != stdout)
745 restore_command (int argc, char *argv[])
748 notmuch_database_t *notmuch = NULL;
757 input = fopen (argv[0], "r");
759 fprintf (stderr, "Error opening %s for reading: %s\n",
760 argv[0], strerror (errno));
765 printf ("No filename given. Reading dump from stdin.\n");
769 notmuch = notmuch_database_open (NULL);
770 if (notmuch == NULL) {
775 /* Dump output is one line per message. We match a sequence of
776 * non-space characters for the message-id, then one or more
777 * spaces, then a list of space-separated tags as a sequence of
778 * characters within literal '(' and ')'. */
780 "^([^ ]+) \\(([^)]*)\\)$",
783 while ((line_len = getline (&line, &line_size, input)) != -1) {
785 char *message_id, *tags, *tag, *next;
786 notmuch_message_t *message;
787 notmuch_status_t status;
789 chomp_newline (line);
791 rerr = xregexec (®ex, line, 3, match, 0);
792 if (rerr == REG_NOMATCH)
794 fprintf (stderr, "Warning: Ignoring invalid input line: %s\n",
799 message_id = xstrndup (line + match[1].rm_so,
800 match[1].rm_eo - match[1].rm_so);
801 tags = xstrndup (line + match[2].rm_so,
802 match[2].rm_eo - match[2].rm_so);
806 message = notmuch_database_find_message (notmuch, message_id);
807 if (message == NULL) {
808 fprintf (stderr, "Warning: Cannot apply tags to missing message: %s (",
814 tag = strsep (&next, " ");
818 status = notmuch_message_add_tag (message, tag);
821 "Error applying tag %s to message %s:\n",
823 fprintf (stderr, "%s\n",
824 notmuch_status_to_string (status));
827 fprintf (stderr, "%s%s",
828 tag == tags ? "" : " ", tag);
833 notmuch_message_destroy (message);
835 fprintf (stderr, ")\n");
847 notmuch_database_close (notmuch);
848 if (input && input != stdin)
854 command_t commands[] = {
855 { "setup", setup_command,
856 "Interactively setup notmuch for first use.\n\n"
857 "\t\tInvoking notmuch with no command argument will run setup if\n"
858 "\t\tthe setup command has not previously been completed." },
859 { "new", new_command,
860 "Find and import any new messages.\n\n"
861 "\t\tScans all sub-directories of the database, adding new files\n"
862 "\t\tthat are found. Note: \"notmuch new\" will skip any\n"
863 "\t\tread-only directories, so you can use that to mark\n"
864 "\t\tdirectories that will not receive any new mail."},
865 { "search", search_command,
866 "<search-term> [...]\n\n"
867 "\t\tSearch for threads matching the given search terms.\n"
868 "\t\tOnce we actually implement search we'll document the\n"
869 "\t\tsyntax here." },
870 { "show", show_command,
872 "\t\tShow the thread with the given thread ID (see 'search')." },
873 { "dump", dump_command,
875 "\t\tCreate a plain-text dump of the tags for each message\n"
876 "\t\twriting to the given filename, if any, or to stdout.\n"
877 "\t\tThese tags are the only data in the notmuch database\n"
878 "\t\tthat can't be recreated from the messages themselves.\n"
879 "\t\tThe output of notmuch dump is therefore the only\n"
880 "\t\tcritical thing to backup (and much more friendly to\n"
881 "\t\tincremental backup than the native database files." },
882 { "restore", restore_command,
884 "\t\tRestore the tags from the given dump file (see 'dump')." }
893 fprintf (stderr, "Usage: notmuch <command> [args...]\n");
894 fprintf (stderr, "\n");
895 fprintf (stderr, "Where <command> and [args...] are as follows:\n");
896 fprintf (stderr, "\n");
898 for (i = 0; i < ARRAY_SIZE (commands); i++) {
899 command = &commands[i];
901 fprintf (stderr, "\t%s\t%s\n\n", command->name, command->usage);
906 main (int argc, char *argv[])
912 return setup_command (0, NULL);
914 for (i = 0; i < ARRAY_SIZE (commands); i++) {
915 command = &commands[i];
917 if (strcmp (argv[1], command->name) == 0)
918 return (command->function) (argc - 2, &argv[2]);
921 /* Don't complain about "help" being an unknown command when we're
922 about to provide exactly what's wanted anyway. */
923 if (strcmp (argv[1], "help") == 0 ||
924 strcmp (argv[1], "--help") == 0)
926 fprintf (stderr, "The notmuch mail system.\n\n");
928 fprintf (stderr, "Error: Unknown command '%s'\n\n", argv[1]);