1 /* notmuch - Not much of an email program, (just index and search)
3 * Copyright © 2009 Carl Worth
5 * This program is free software: you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation, either version 3 of the License, or
8 * (at your option) any later version.
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
15 * You should have received a copy of the GNU General Public License
16 * along with this program. If not, see http://www.gnu.org/licenses/ .
18 * Author: Carl Worth <cworth@cworth.org>
22 #define _GNU_SOURCE /* for getline */
28 /* This is separate from notmuch-private.h because we're trying to
29 * keep notmuch.c from looking into any internals, (which helps us
30 * develop notmuch.h into a plausible library interface).
44 #include <glib.h> /* g_strdup_printf */
46 #define ARRAY_SIZE(arr) (sizeof (arr) / sizeof (arr[0]))
48 typedef int (*command_function_t) (int argc, char *argv[]);
50 typedef struct command {
52 command_function_t function;
57 int ignore_read_only_directories;
62 struct timeval tv_start;
66 chomp_newline (char *str)
68 if (str && str[strlen(str)-1] == '\n')
69 str[strlen(str)-1] = '\0';
72 /* Compute the number of seconds elapsed from start to end. */
74 tv_elapsed (struct timeval start, struct timeval end)
76 return ((end.tv_sec - start.tv_sec) +
77 (end.tv_usec - start.tv_usec) / 1e6);
81 print_formatted_seconds (double seconds)
87 printf ("almost no time");
92 hours = (int) seconds / 3600;
93 printf ("%dh ", hours);
94 seconds -= hours * 3600;
98 minutes = (int) seconds / 60;
99 printf ("%dm ", minutes);
100 seconds -= minutes * 60;
103 printf ("%ds", (int) seconds);
107 add_files_print_progress (add_files_state_t *state)
109 struct timeval tv_now;
110 double elapsed_overall, rate_overall;
112 gettimeofday (&tv_now, NULL);
114 elapsed_overall = tv_elapsed (state->tv_start, tv_now);
115 rate_overall = (state->processed_files) / elapsed_overall;
117 printf ("Processed %d", state->processed_files);
119 if (state->total_files) {
120 printf (" of %d files (", state->total_files);
121 print_formatted_seconds ((state->total_files - state->processed_files) /
123 printf (" remaining). \r");
125 printf (" files (%d files/sec.) \r", (int) rate_overall);
131 /* Examine 'path' recursively as follows:
133 * o Ask the filesystem for the mtime of 'path' (path_mtime)
135 * o Ask the database for its timestamp of 'path' (path_dbtime)
137 * o If 'path_mtime' > 'path_dbtime'
139 * o For each regular file in 'path' with mtime newer than the
140 * 'path_dbtime' call add_message to add the file to the
143 * o For each sub-directory of path, recursively call into this
146 * o Tell the database to update its time of 'path' to 'path_mtime'
148 * The 'struct stat *st' must point to a structure that has already
149 * been initialized for 'path' by calling stat().
152 add_files_recursive (notmuch_database_t *notmuch,
155 add_files_state_t *state)
158 struct dirent *e, *entry = NULL;
162 time_t path_mtime, path_dbtime;
163 notmuch_status_t status, ret = NOTMUCH_STATUS_SUCCESS;
165 /* If we're told to, we bail out on encountering a read-only
166 * directory, (with this being a clear clue from the user to
167 * Notmuch that new mail won't be arriving there and we need not
169 if (state->ignore_read_only_directories &&
170 (st->st_mode & S_IWUSR) == 0)
175 path_mtime = st->st_mtime;
177 path_dbtime = notmuch_database_get_timestamp (notmuch, path);
179 dir = opendir (path);
181 fprintf (stderr, "Error opening directory %s: %s\n",
182 path, strerror (errno));
183 ret = NOTMUCH_STATUS_FILE_ERROR;
187 entry_length = offsetof (struct dirent, d_name) +
188 pathconf (path, _PC_NAME_MAX) + 1;
189 entry = malloc (entry_length);
192 err = readdir_r (dir, entry, &e);
194 fprintf (stderr, "Error reading directory: %s\n",
196 ret = NOTMUCH_STATUS_FILE_ERROR;
203 /* If this directory hasn't been modified since the last
204 * add_files, then we only need to look further for
205 * sub-directories. */
206 if (path_mtime <= path_dbtime && entry->d_type != DT_DIR)
209 /* Ignore special directories to avoid infinite recursion.
210 * Also ignore the .notmuch directory.
212 /* XXX: Eventually we'll want more sophistication to let the
213 * user specify files to be ignored. */
214 if (strcmp (entry->d_name, ".") == 0 ||
215 strcmp (entry->d_name, "..") == 0 ||
216 strcmp (entry->d_name, ".notmuch") ==0)
221 next = g_strdup_printf ("%s/%s", path, entry->d_name);
223 if (stat (next, st)) {
224 fprintf (stderr, "Error reading %s: %s\n",
225 next, strerror (errno));
226 ret = NOTMUCH_STATUS_FILE_ERROR;
230 if (S_ISREG (st->st_mode)) {
231 /* If the file hasn't been modified since the last
232 * add_files, then we need not look at it. */
233 if (st->st_mtime > path_dbtime) {
234 state->processed_files++;
236 status = notmuch_database_add_message (notmuch, next);
239 case NOTMUCH_STATUS_SUCCESS:
240 state->added_messages++;
242 /* Non-fatal issues (go on to next file) */
243 case NOTMUCH_STATUS_DUPLICATE_MESSAGE_ID:
244 /* Stay silent on this one. */
246 case NOTMUCH_STATUS_FILE_NOT_EMAIL:
247 fprintf (stderr, "Note: Ignoring non-mail file: %s\n",
250 /* Fatal issues. Don't process anymore. */
251 case NOTMUCH_STATUS_XAPIAN_EXCEPTION:
252 fprintf (stderr, "A Xapian error was encountered. Halting processing.\n");
256 fprintf (stderr, "Internal error: add_message returned unexpected value: %d\n", status);
260 if (state->processed_files % 1000 == 0)
261 add_files_print_progress (state);
263 } else if (S_ISDIR (st->st_mode)) {
264 status = add_files_recursive (notmuch, next, st, state);
265 if (status && ret == NOTMUCH_STATUS_SUCCESS)
273 status = notmuch_database_set_timestamp (notmuch, path, path_mtime);
274 if (status && ret == NOTMUCH_STATUS_SUCCESS)
288 /* This is the top-level entry point for add_files. It does a couple
289 * of error checks, and then calls into the recursive function,
290 * (avoiding the repeating of these error checks at every
291 * level---which would be useless becaues we already do a stat() at
292 * the level above). */
293 static notmuch_status_t
294 add_files (notmuch_database_t *notmuch,
296 add_files_state_t *state)
300 if (stat (path, &st)) {
301 fprintf (stderr, "Error reading directory %s: %s\n",
302 path, strerror (errno));
303 return NOTMUCH_STATUS_FILE_ERROR;
306 if (! S_ISDIR (st.st_mode)) {
307 fprintf (stderr, "Error: %s is not a directory.\n", path);
308 return NOTMUCH_STATUS_FILE_ERROR;
311 return add_files_recursive (notmuch, path, &st, state);
314 /* Recursively count all regular files in path and all sub-direcotries
315 * of path. The result is added to *count (which should be
316 * initialized to zero by the top-level caller before calling
319 count_files (const char *path, int *count)
322 struct dirent *entry, *e;
328 dir = opendir (path);
331 fprintf (stderr, "Warning: failed to open directory %s: %s\n",
332 path, strerror (errno));
336 entry_length = offsetof (struct dirent, d_name) +
337 pathconf (path, _PC_NAME_MAX) + 1;
338 entry = malloc (entry_length);
341 err = readdir_r (dir, entry, &e);
343 fprintf (stderr, "Error reading directory: %s\n",
352 /* Ignore special directories to avoid infinite recursion.
353 * Also ignore the .notmuch directory.
355 /* XXX: Eventually we'll want more sophistication to let the
356 * user specify files to be ignored. */
357 if (strcmp (entry->d_name, ".") == 0 ||
358 strcmp (entry->d_name, "..") == 0 ||
359 strcmp (entry->d_name, ".notmuch") == 0)
364 next = g_strdup_printf ("%s/%s", path, entry->d_name);
368 if (S_ISREG (st.st_mode)) {
370 if (*count % 1000 == 0) {
371 printf ("Found %d files so far.\r", *count);
374 } else if (S_ISDIR (st.st_mode)) {
375 count_files (next, count);
387 setup_command (int argc, char *argv[])
389 notmuch_database_t *notmuch = NULL;
390 char *default_path, *mail_directory = NULL;
393 add_files_state_t add_files_state;
395 struct timeval tv_now;
396 notmuch_status_t ret = NOTMUCH_STATUS_SUCCESS;
398 printf ("Welcome to notmuch!\n\n");
400 printf ("The goal of notmuch is to help you manage and search your collection of\n"
401 "email, and to efficiently keep up with the flow of email as it comes in.\n\n");
403 printf ("Notmuch needs to know the top-level directory of your email archive,\n"
404 "(where you already have mail stored and where messages will be delivered\n"
405 "in the future). This directory can contain any number of sub-directories\n"
406 "and primarily just files with indvidual email messages (eg. maildir or mh\n"
407 "archives are perfect). If there are other, non-email files (such as\n"
408 "indexes maintained by other email programs) then notmuch will do its\n"
409 "best to detect those and ignore them.\n\n");
411 printf ("Mail storage that uses mbox format, (where one mbox file contains many\n"
412 "messages), will not work with notmuch. If that's how your mail is currently\n"
413 "stored, we recommend you first convert it to maildir format with a utility\n"
414 "such as mb2md. In that case, press Control-C now and run notmuch again\n"
415 "once the conversion is complete.\n\n");
418 default_path = notmuch_database_default_path ();
419 printf ("Top-level mail directory [%s]: ", default_path);
422 getline (&mail_directory, &line_size, stdin);
423 chomp_newline (mail_directory);
427 if (mail_directory == NULL || strlen (mail_directory) == 0) {
429 free (mail_directory);
430 mail_directory = default_path;
432 /* XXX: Instead of telling the user to use an environment
433 * variable here, we should really be writing out a configuration
434 * file and loading that on the next run. */
435 if (strcmp (mail_directory, default_path)) {
436 printf ("Note: Since you are not using the default path, you will want to set\n"
437 "the NOTMUCH_BASE environment variable to %s so that\n"
438 "future calls to notmuch commands will know where to find your mail.\n",
440 printf ("For example, if you are using bash for your shell, add:\n\n");
441 printf ("\texport NOTMUCH_BASE=%s\n\n", mail_directory);
442 printf ("to your ~/.bashrc file.\n\n");
447 notmuch = notmuch_database_create (mail_directory);
448 if (notmuch == NULL) {
449 fprintf (stderr, "Failed to create new notmuch database at %s\n",
451 ret = NOTMUCH_STATUS_FILE_ERROR;
455 printf ("OK. Let's take a look at the mail we can find in the directory\n");
456 printf ("%s ...\n", mail_directory);
459 count_files (mail_directory, &count);
461 printf ("Found %d total files. That's not much mail.\n\n", count);
463 printf ("Next, we'll inspect the messages and create a database of threads:\n");
465 add_files_state.ignore_read_only_directories = FALSE;
466 add_files_state.total_files = count;
467 add_files_state.processed_files = 0;
468 add_files_state.added_messages = 0;
469 gettimeofday (&add_files_state.tv_start, NULL);
471 ret = add_files (notmuch, mail_directory, &add_files_state);
473 gettimeofday (&tv_now, NULL);
474 elapsed = tv_elapsed (add_files_state.tv_start,
476 printf ("Processed %d %s in ", add_files_state.processed_files,
477 add_files_state.processed_files == 1 ?
478 "file" : "total files");
479 print_formatted_seconds (elapsed);
481 printf (" (%d files/sec.). \n",
482 (int) (add_files_state.processed_files / elapsed));
486 if (add_files_state.added_messages) {
487 printf ("Added %d %s to the database.\n\n",
488 add_files_state.added_messages,
489 add_files_state.added_messages == 1 ?
490 "message" : "unique messages");
493 printf ("When new mail is delivered to %s in the future,\n"
494 "run \"notmuch new\" to add it to the database.\n\n",
496 printf ("And if you have any sub-directories that are archives (that is,\n"
497 "they will never receive new mail), marking these directores as\n"
498 "read-only (chmod u-w /path/to/dir) will make \"notmuch new\"\n"
499 "much more efficient (it won't even look in those directories).\n\n");
502 printf ("Note: At least one error was encountered: %s\n",
503 notmuch_status_to_string (ret));
508 free (mail_directory);
510 notmuch_database_close (notmuch);
516 new_command (int argc, char *argv[])
518 notmuch_database_t *notmuch;
519 const char *mail_directory;
520 add_files_state_t add_files_state;
522 struct timeval tv_now;
525 notmuch = notmuch_database_open (NULL);
526 if (notmuch == NULL) {
531 mail_directory = notmuch_database_get_path (notmuch);
533 add_files_state.ignore_read_only_directories = TRUE;
534 add_files_state.total_files = 0;
535 add_files_state.processed_files = 0;
536 add_files_state.added_messages = 0;
537 gettimeofday (&add_files_state.tv_start, NULL);
539 ret = add_files (notmuch, mail_directory, &add_files_state);
541 gettimeofday (&tv_now, NULL);
542 elapsed = tv_elapsed (add_files_state.tv_start,
544 if (add_files_state.processed_files) {
545 printf ("Processed %d %s in ", add_files_state.processed_files,
546 add_files_state.processed_files == 1 ?
547 "file" : "total files");
548 print_formatted_seconds (elapsed);
550 printf (" (%d files/sec.). \n",
551 (int) (add_files_state.processed_files / elapsed));
556 if (add_files_state.added_messages) {
557 printf ("Added %d new %s to the database (not much, really).\n",
558 add_files_state.added_messages,
559 add_files_state.added_messages == 1 ?
560 "message" : "messages");
562 printf ("No new mail---and that's not much.\n");
566 printf ("Note: At least one error was encountered: %s\n",
567 notmuch_status_to_string (ret));
572 notmuch_database_close (notmuch);
578 search_command (int argc, char *argv[])
580 void *local = talloc_new (NULL);
581 notmuch_database_t *notmuch = NULL;
582 notmuch_query_t *query;
583 notmuch_results_t *results;
584 notmuch_message_t *message;
585 notmuch_tags_t *tags;
588 notmuch_status_t ret = NOTMUCH_STATUS_SUCCESS;
590 notmuch = notmuch_database_open (NULL);
591 if (notmuch == NULL) {
596 /* XXX: Should add xtalloc wrappers here and use them. */
597 query_str = talloc_strdup (local, "");
599 for (i = 0; i < argc; i++) {
601 query_str = talloc_asprintf_append (query_str, " ");
603 query_str = talloc_asprintf_append (query_str, "%s", argv[i]);
606 query = notmuch_query_create (notmuch, query_str);
608 fprintf (stderr, "Out of memory\n");
613 for (results = notmuch_query_search (query);
614 notmuch_results_has_more (results);
615 notmuch_results_advance (results))
618 message = notmuch_results_get (results);
620 printf ("%s (", notmuch_message_get_message_id (message));
622 for (tags = notmuch_message_get_tags (message);
623 notmuch_tags_has_more (tags);
624 notmuch_tags_advance (tags))
629 printf ("%s", notmuch_tags_get (tags));
636 notmuch_message_destroy (message);
639 notmuch_query_destroy (query);
643 notmuch_database_close (notmuch);
650 show_command (int argc, char *argv[])
652 fprintf (stderr, "Error: show is not implemented yet.\n");
657 dump_command (int argc, char *argv[])
660 notmuch_database_t *notmuch = NULL;
661 notmuch_query_t *query;
662 notmuch_results_t *results;
663 notmuch_message_t *message;
664 notmuch_tags_t *tags;
668 output = fopen (argv[0], "w");
669 if (output == NULL) {
670 fprintf (stderr, "Error opening %s for writing: %s\n",
671 argv[0], strerror (errno));
679 notmuch = notmuch_database_open (NULL);
680 if (notmuch == NULL) {
685 query = notmuch_query_create (notmuch, "");
687 fprintf (stderr, "Out of memory\n");
692 notmuch_query_set_sort (query, NOTMUCH_SORT_MESSAGE_ID);
694 for (results = notmuch_query_search (query);
695 notmuch_results_has_more (results);
696 notmuch_results_advance (results))
699 message = notmuch_results_get (results);
702 "%s (", notmuch_message_get_message_id (message));
704 for (tags = notmuch_message_get_tags (message);
705 notmuch_tags_has_more (tags);
706 notmuch_tags_advance (tags))
709 fprintf (output, " ");
711 fprintf (output, "%s", notmuch_tags_get (tags));
716 fprintf (output, ")\n");
718 notmuch_message_destroy (message);
721 notmuch_query_destroy (query);
725 notmuch_database_close (notmuch);
726 if (output != stdout)
733 restore_command (int argc, char *argv[])
736 notmuch_database_t *notmuch = NULL;
738 size_t line_size, line_len;
744 input = fopen (argv[0], "r");
746 fprintf (stderr, "Error opening %s for reading: %s\n",
747 argv[0], strerror (errno));
752 printf ("No filename given. Reading dump from stdin.\n");
756 notmuch = notmuch_database_open (NULL);
757 if (notmuch == NULL) {
762 /* Dump output is one line per message. We match a sequence of
763 * non-space characters for the message-id, then one or more
764 * spaces, then a list of space-separated tags as a sequence of
765 * characters within literal '(' and ')'. */
767 "^([^ ]+) \\(([^)]*)\\)$",
770 while ((line_len = getline (&line, &line_size, input)) != -1) {
772 char *message_id, *tags, *tag, *next;
773 notmuch_message_t *message;
774 notmuch_status_t status;
776 chomp_newline (line);
778 rerr = xregexec (®ex, line, 3, match, 0);
779 if (rerr == REG_NOMATCH)
781 fprintf (stderr, "Warning: Ignoring invalid input line: %s\n",
786 message_id = xstrndup (line + match[1].rm_so,
787 match[1].rm_eo - match[1].rm_so);
788 tags = xstrndup (line + match[2].rm_so,
789 match[2].rm_eo - match[2].rm_so);
793 message = notmuch_database_find_message (notmuch, message_id);
794 if (message == NULL) {
795 fprintf (stderr, "Warning: Cannot apply tags to missing message: %s (",
801 tag = strsep (&next, " ");
805 status = notmuch_message_add_tag (message, tag);
808 "Error applying tag %s to message %s:\n",
810 fprintf (stderr, "%s\n",
811 notmuch_status_to_string (status));
814 fprintf (stderr, "%s ", tag);
819 notmuch_message_destroy (message);
821 fprintf (stderr, ")\n");
833 notmuch_database_close (notmuch);
838 command_t commands[] = {
839 { "setup", setup_command,
840 "Interactively setup notmuch for first use.\n\n"
841 "\t\tInvoking notmuch with no command argument will run setup if\n"
842 "\t\tthe setup command has not previously been completed." },
843 { "new", new_command,
844 "Find and import any new messages.\n\n"
845 "\t\tScans all sub-directories of the database, adding new files\n"
846 "\t\tthat are found. Note: \"notmuch new\" will skip any\n"
847 "\t\tread-only directories, so you can use that to mark\n"
848 "\t\tdirectories that will not receive any new mail."},
849 { "search", search_command,
850 "<search-term> [...]\n\n"
851 "\t\tSearch for threads matching the given search terms.\n"
852 "\t\tOnce we actually implement search we'll document the\n"
853 "\t\tsyntax here." },
854 { "show", show_command,
856 "\t\tShow the thread with the given thread ID (see 'search')." },
857 { "dump", dump_command,
859 "\t\tCreate a plain-text dump of the tags for each message\n"
860 "\t\twriting to the given filename, if any, or to stdout.\n"
861 "\t\tThese tags are the only data in the notmuch database\n"
862 "\t\tthat can't be recreated from the messages themselves.\n"
863 "\t\tThe output of notmuch dump is therefore the only\n"
864 "\t\tcritical thing to backup (and much more friendly to\n"
865 "\t\tincremental backup than the native database files." },
866 { "restore", restore_command,
868 "\t\tRestore the tags from the given dump file (see 'dump')." }
877 fprintf (stderr, "Usage: notmuch <command> [args...]\n");
878 fprintf (stderr, "\n");
879 fprintf (stderr, "Where <command> and [args...] are as follows:\n");
880 fprintf (stderr, "\n");
882 for (i = 0; i < ARRAY_SIZE (commands); i++) {
883 command = &commands[i];
885 fprintf (stderr, "\t%s\t%s\n\n", command->name, command->usage);
890 main (int argc, char *argv[])
896 return setup_command (0, NULL);
898 for (i = 0; i < ARRAY_SIZE (commands); i++) {
899 command = &commands[i];
901 if (strcmp (argv[1], command->name) == 0)
902 return (command->function) (argc - 2, &argv[2]);
905 /* Don't complain about "help" being an unknown command when we're
906 about to provide exactly what's wanted anyway. */
907 if (strcmp (argv[1], "help") == 0 ||
908 strcmp (argv[1], "--help") == 0)
910 fprintf (stderr, "The notmuch mail system.\n\n");
912 fprintf (stderr, "Error: Unknown command '%s'\n\n", argv[1]);