From e6236b88fd18231d0524b14723e0709a90b0572c Mon Sep 17 00:00:00 2001 From: Carl Worth Date: Tue, 20 Oct 2009 12:49:32 -0700 Subject: [PATCH 01/16] database.cc: Document better pieces of glib that we're using. --- database.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/database.cc b/database.cc index a5365103..abae5820 100644 --- a/database.cc +++ b/database.cc @@ -24,7 +24,7 @@ #include -#include /* g_strdup_printf, g_free, GHashTable */ +#include /* g_strdup_printf, g_free, GPtrArray, GHashTable */ using namespace std; -- 2.43.0 From 5a84df0f1507753b1776636ad268d36b7346a8fd Mon Sep 17 00:00:00 2001 From: Carl Worth Date: Tue, 20 Oct 2009 13:05:45 -0700 Subject: [PATCH 02/16] add_message: Fix memory leak of thread_ids GPtrArray. We were properly feeing this memory when the thread-ids list was not empty, but leaking it when it was. Thanks, of course, to valgrind along with the G_SLICE=always-malloc environment variable which makes leak checking with glib almost bearable. --- database.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/database.cc b/database.cc index abae5820..59e83736 100644 --- a/database.cc +++ b/database.cc @@ -577,7 +577,6 @@ notmuch_database_add_message (notmuch_database_t *notmuch, free (id); } - g_ptr_array_free (thread_ids, TRUE); doc.add_value (NOTMUCH_VALUE_THREAD, thread_id->str); g_string_free (thread_id, TRUE); } else if (message_id) { @@ -589,6 +588,8 @@ notmuch_database_add_message (notmuch_database_t *notmuch, doc.add_value (NOTMUCH_VALUE_THREAD, thread_id.str); } + g_ptr_array_free (thread_ids, TRUE); + free (message_id); date = notmuch_message_get_header (message, "date"); -- 2.43.0 From 7f254fb603f5ee073d48b052c00e1fdb6b8dc940 Mon Sep 17 00:00:00 2001 From: Carl Worth Date: Tue, 20 Oct 2009 13:07:19 -0700 Subject: [PATCH 03/16] message: Use g_hash_table_destroy instead of g_hash_table_unref I'm trying to chase down 3 still-reachable pointers to glib hash tables. This change didn't help with that, but I think destroy might be a better semantic match for what I actually want. (It shouldn't matter though since I never take any additional references.) --- message.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/message.c b/message.c index e6488a37..d888b24d 100644 --- a/message.c +++ b/message.c @@ -111,7 +111,7 @@ notmuch_message_close (notmuch_message_t *message) free (message->value.str); if (message->headers) - g_hash_table_unref (message->headers); + g_hash_table_destroy (message->headers); if (message->file) fclose (message->file); -- 2.43.0 From 28fa0bc2d686761f50f5bd6782b81000c45ee0c4 Mon Sep 17 00:00:00 2001 From: Carl Worth Date: Tue, 20 Oct 2009 13:16:16 -0700 Subject: [PATCH 04/16] notmuch: Fix setup so that accepting the default mail path works. The recent change from GIOChannel to getline, (with a semantic change of the newline terminator now being included in the result that setup_command sees), broke this. --- notmuch.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/notmuch.c b/notmuch.c index 966901f0..1ebd613e 100644 --- a/notmuch.c +++ b/notmuch.c @@ -289,13 +289,17 @@ setup_command (int argc, char *argv[]) getline (&mail_directory, &line_size, stdin); printf ("\n"); + if (mail_directory && + mail_directory[strlen(mail_directory)-1] == '\n') + { + mail_directory[strlen(mail_directory)-1] = '\0'; + } + if (mail_directory == NULL || strlen (mail_directory) == 0) { if (mail_directory) free (mail_directory); mail_directory = default_path; } else { - if (mail_directory[strlen(mail_directory)-1] == '\n') - mail_directory[strlen(mail_directory)-1] = '\0'; /* XXX: Instead of telling the user to use an environment * variable here, we should really be writing out a configuration * file and loading that on the next run. */ -- 2.43.0 From 00af443b8e5814f26c87c7cd8db1a08fec71e0b3 Mon Sep 17 00:00:00 2001 From: Carl Worth Date: Tue, 20 Oct 2009 15:08:03 -0700 Subject: [PATCH 05/16] Makefile: Add automatic dependency tracking to the Makefile. With this, I really don't miss anything from automake. --- Makefile | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 7c15d6c1..86516402 100644 --- a/Makefile +++ b/Makefile @@ -8,13 +8,17 @@ MYLDFLAGS=`pkg-config --libs glib-2.0` `xapian-config --libs` all: $(PROGS) %.o: %.cc - $(CXX) -c $(CXXFLAGS) $(MYCXXFLAGS) $^ -o $@ + $(CXX) -c $(CXXFLAGS) $(MYCXXFLAGS) $< -o $@ %.o: %.c - $(CC) -c $(CFLAGS) $(MYCFLAGS) $^ -o $@ + $(CC) -c $(CFLAGS) $(MYCFLAGS) $< -o $@ notmuch: notmuch.o database.o date.o message.o xutil.o $(CC) $(MYLDFLAGS) $^ -o $@ +Makefile.dep: *.c *.cc + $(CC) -M $(CPPFLAGS) $(MYCFLAGS) $^ > $@ +-include Makefile.dep + clean: - rm -f $(PROGS) *.o + rm -f $(PROGS) *.o Makefile.dep -- 2.43.0 From cd4a8734d3bb151df70d51a84903bff994439b05 Mon Sep 17 00:00:00 2001 From: Carl Worth Date: Tue, 20 Oct 2009 15:09:51 -0700 Subject: [PATCH 06/16] Rename private notmuch_message_t to notmuch_message_file_t This is in preparation for a new, public notmuch_message_t. Eventually, the public notmuch_message_t is going to grow enough features to need to be file-backed and will likely need everything that's now in message-file.c. So we may fold these back into one object/implementation in the future. --- Makefile | 2 +- database.cc | 38 ++++++++++++++++++------------------- message.c => message-file.c | 28 +++++++++++++-------------- notmuch-private.h | 20 +++++++++---------- 4 files changed, 44 insertions(+), 44 deletions(-) rename message.c => message-file.c (88%) diff --git a/Makefile b/Makefile index 86516402..34716c29 100644 --- a/Makefile +++ b/Makefile @@ -13,7 +13,7 @@ all: $(PROGS) %.o: %.c $(CC) -c $(CFLAGS) $(MYCFLAGS) $< -o $@ -notmuch: notmuch.o database.o date.o message.o xutil.o +notmuch: notmuch.o database.o date.o message-file.o xutil.o $(CC) $(MYLDFLAGS) $^ -o $@ Makefile.dep: *.c *.cc diff --git a/database.cc b/database.cc index 59e83736..7e678d87 100644 --- a/database.cc +++ b/database.cc @@ -501,7 +501,7 @@ notmuch_database_add_message (notmuch_database_t *notmuch, { Xapian::WritableDatabase *db = notmuch->xapian_db; Xapian::Document doc; - notmuch_message_t *message; + notmuch_message_file_t *message; GPtrArray *parents, *thread_ids; @@ -512,16 +512,16 @@ notmuch_database_add_message (notmuch_database_t *notmuch, time_t time_value; unsigned int i; - message = notmuch_message_open (filename); + message = notmuch_message_file_open (filename); - notmuch_message_restrict_headers (message, - "date", - "from", - "in-reply-to", - "message-id", - "references", - "subject", - (char *) NULL); + notmuch_message_file_restrict_headers (message, + "date", + "from", + "in-reply-to", + "message-id", + "references", + "subject", + (char *) NULL); try { doc = Xapian::Document (); @@ -530,16 +530,16 @@ notmuch_database_add_message (notmuch_database_t *notmuch, parents = g_ptr_array_new (); - refs = notmuch_message_get_header (message, "references"); + refs = notmuch_message_file_get_header (message, "references"); parse_references (parents, refs); - in_reply_to = notmuch_message_get_header (message, "in-reply-to"); + in_reply_to = notmuch_message_file_get_header (message, "in-reply-to"); parse_references (parents, in_reply_to); for (i = 0; i < parents->len; i++) add_term (doc, "ref", (char *) g_ptr_array_index (parents, i)); - header = notmuch_message_get_header (message, "message-id"); + header = notmuch_message_file_get_header (message, "message-id"); if (header) { message_id = parse_message_id (header, NULL); /* So the header value isn't RFC-compliant, but it's @@ -592,21 +592,21 @@ notmuch_database_add_message (notmuch_database_t *notmuch, free (message_id); - date = notmuch_message_get_header (message, "date"); + date = notmuch_message_file_get_header (message, "date"); time_value = notmuch_parse_date (date, NULL); doc.add_value (NOTMUCH_VALUE_DATE, Xapian::sortable_serialise (time_value)); - from = notmuch_message_get_header (message, "from"); - subject = notmuch_message_get_header (message, "subject"); - to = notmuch_message_get_header (message, "to"); + from = notmuch_message_file_get_header (message, "from"); + subject = notmuch_message_file_get_header (message, "subject"); + to = notmuch_message_file_get_header (message, "to"); if (from == NULL && subject == NULL && to == NULL) { - notmuch_message_close (message); + notmuch_message_file_close (message); return NOTMUCH_STATUS_FILE_NOT_EMAIL; } else { db->add_document (doc); @@ -617,7 +617,7 @@ notmuch_database_add_message (notmuch_database_t *notmuch, return NOTMUCH_STATUS_XAPIAN_EXCEPTION; } - notmuch_message_close (message); + notmuch_message_file_close (message); return NOTMUCH_STATUS_SUCCESS; } diff --git a/message.c b/message-file.c similarity index 88% rename from message.c rename to message-file.c index d888b24d..4f4f551b 100644 --- a/message.c +++ b/message-file.c @@ -30,7 +30,7 @@ typedef struct { size_t len; } header_value_closure_t; -struct _notmuch_message { +struct _notmuch_message_file { /* File object */ FILE *file; @@ -70,12 +70,12 @@ strcase_hash (const void *ptr) return hash; } -notmuch_message_t * -notmuch_message_open (const char *filename) +notmuch_message_file_t * +notmuch_message_file_open (const char *filename) { - notmuch_message_t *message; + notmuch_message_file_t *message; - message = xcalloc (1, sizeof (notmuch_message_t)); + message = xcalloc (1, sizeof (notmuch_message_file_t)); message->file = fopen (filename, "r"); if (message->file == NULL) @@ -93,13 +93,13 @@ notmuch_message_open (const char *filename) FAIL: fprintf (stderr, "Error opening %s: %s\n", filename, strerror (errno)); - notmuch_message_close (message); + notmuch_message_file_close (message); return NULL; } void -notmuch_message_close (notmuch_message_t *message) +notmuch_message_file_close (notmuch_message_file_t *message) { if (message == NULL) return; @@ -120,13 +120,13 @@ notmuch_message_close (notmuch_message_t *message) } void -notmuch_message_restrict_headersv (notmuch_message_t *message, - va_list va_headers) +notmuch_message_file_restrict_headersv (notmuch_message_file_t *message, + va_list va_headers) { char *header; if (message->parsing_started ) { - fprintf (stderr, "Error: notmuch_message_restrict_headers called after parsing has started\n"); + fprintf (stderr, "Error: notmuch_message_file_restrict_headers called after parsing has started\n"); exit (1); } @@ -142,13 +142,13 @@ notmuch_message_restrict_headersv (notmuch_message_t *message, } void -notmuch_message_restrict_headers (notmuch_message_t *message, ...) +notmuch_message_file_restrict_headers (notmuch_message_file_t *message, ...) { va_list va_headers; va_start (va_headers, message); - notmuch_message_restrict_headersv (message, va_headers); + notmuch_message_file_restrict_headersv (message, va_headers); } void @@ -192,8 +192,8 @@ copy_header_unfolding (header_value_closure_t *value, } const char * -notmuch_message_get_header (notmuch_message_t *message, - const char *header_desired) +notmuch_message_file_get_header (notmuch_message_file_t *message, + const char *header_desired) { int contains; char *header, *value; diff --git a/notmuch-private.h b/notmuch-private.h index a8ec8ebe..0c20a3c4 100644 --- a/notmuch-private.h +++ b/notmuch-private.h @@ -57,13 +57,13 @@ xstrdup (const char *s); char * xstrndup (const char *s, size_t n); -/* message.c */ +/* message-file.c */ /* XXX: I haven't decided yet whether these will actually get exported * into the public interface in notmuch.h */ -typedef struct _notmuch_message notmuch_message_t; +typedef struct _notmuch_message_file notmuch_message_file_t; /* Open a file containing a single email message. * @@ -71,12 +71,12 @@ typedef struct _notmuch_message notmuch_message_t; * * Returns NULL if any error occurs. */ -notmuch_message_t * -notmuch_message_open (const char *filename); +notmuch_message_file_t * +notmuch_message_file_open (const char *filename); /* Close a notmuch message preivously opened with notmuch_message_open. */ void -notmuch_message_close (notmuch_message_t *message); +notmuch_message_file_close (notmuch_message_file_t *message); /* Restrict 'message' to only save the named headers. * @@ -95,12 +95,12 @@ notmuch_message_close (notmuch_message_t *message); * returned even if that header exists in the actual message. */ void -notmuch_message_restrict_headers (notmuch_message_t *message, ...); +notmuch_message_file_restrict_headers (notmuch_message_file_t *message, ...); /* Identical to notmuch_message_restrict_headers but accepting a va_list. */ void -notmuch_message_restrict_headersv (notmuch_message_t *message, - va_list va_headers); +notmuch_message_file_restrict_headersv (notmuch_message_file_t *message, + va_list va_headers); /* Get the value of the specified header from the message. * @@ -114,8 +114,8 @@ notmuch_message_restrict_headersv (notmuch_message_t *message, * 'header'. */ const char * -notmuch_message_get_header (notmuch_message_t *message, - const char *header); +notmuch_message_file_get_header (notmuch_message_file_t *message, + const char *header); /* date.c */ -- 2.43.0 From 466a7bbf620e4bf1b57097a6d3c474159c475b6d Mon Sep 17 00:00:00 2001 From: Carl Worth Date: Tue, 20 Oct 2009 21:03:30 -0700 Subject: [PATCH 07/16] Implement 'notmuch dump'. This is a fairly big milestone for notmuch. It's our first command to do anything besides building the index, so it proves we can actually read valid results out from the index. It also puts in place almost all of the API and infrastructure we will need to allow searching of the database. Finally, with this change we are now using talloc inside of notmuch which is truly a delight to use. And now that I figured out how to use C++ objects with talloc allocation, (it requires grotty parts of C++ such as "placement new" and "explicit destructors"), we are valgrind-clean for "notmuch dump", (as in "no leaks are possible"). --- Makefile | 6 +- database-private.h | 34 ++++++++ database.cc | 9 +-- message.cc | 131 ++++++++++++++++++++++++++++++ notmuch-private.h | 32 ++++++++ notmuch.c | 74 ++++++++++++++++- notmuch.h | 198 ++++++++++++++++++++++++++++++++++++++++++++- query.cc | 137 +++++++++++++++++++++++++++++++ 8 files changed, 606 insertions(+), 15 deletions(-) create mode 100644 database-private.h create mode 100644 message.cc create mode 100644 query.cc diff --git a/Makefile b/Makefile index 34716c29..ff654e11 100644 --- a/Makefile +++ b/Makefile @@ -1,9 +1,9 @@ PROGS=notmuch -MYCFLAGS=-Wall -O0 -g `pkg-config --cflags glib-2.0` +MYCFLAGS=-Wall -O0 -g `pkg-config --cflags glib-2.0 talloc` MYCXXFLAGS=$(MYCFLAGS) `xapian-config --cxxflags` -MYLDFLAGS=`pkg-config --libs glib-2.0` `xapian-config --libs` +MYLDFLAGS=`pkg-config --libs glib-2.0 talloc` `xapian-config --libs` all: $(PROGS) @@ -13,7 +13,7 @@ all: $(PROGS) %.o: %.c $(CC) -c $(CFLAGS) $(MYCFLAGS) $< -o $@ -notmuch: notmuch.o database.o date.o message-file.o xutil.o +notmuch: notmuch.o database.o date.o message.o message-file.o query.o xutil.o $(CC) $(MYLDFLAGS) $^ -o $@ Makefile.dep: *.c *.cc diff --git a/database-private.h b/database-private.h new file mode 100644 index 00000000..b894717e --- /dev/null +++ b/database-private.h @@ -0,0 +1,34 @@ +/* database-private.h - For peeking into the internals of notmuch_database_t + * + * Copyright © 2009 Carl Worth + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see http://www.gnu.org/licenses/ . + * + * Author: Carl Worth + */ + +#ifndef NOTMUCH_DATABASE_PRIVATE_H +#define NOTMUCH_DATABASE_PRIVATE_H + +#include "notmuch-private.h" + +#include + +struct _notmuch_database { + char *path; + Xapian::WritableDatabase *xapian_db; + Xapian::TermGenerator *term_gen; +}; + +#endif diff --git a/database.cc b/database.cc index 7e678d87..041cffdc 100644 --- a/database.cc +++ b/database.cc @@ -18,7 +18,7 @@ * Author: Carl Worth */ -#include "notmuch-private.h" +#include "database-private.h" #include @@ -28,12 +28,6 @@ using namespace std; -struct _notmuch_database { - char *path; - Xapian::WritableDatabase *xapian_db; - Xapian::TermGenerator *term_gen; -}; - #define ARRAY_SIZE(arr) (sizeof (arr) / sizeof (arr[0])) /* Xapian complains if we provide a term longer than this. */ @@ -463,6 +457,7 @@ notmuch_database_open (const char *path) notmuch->path = xstrdup (path); try { + Xapian::PostingIterator i; notmuch->xapian_db = new Xapian::WritableDatabase (xapian_path, Xapian::DB_CREATE_OR_OPEN); } catch (const Xapian::Error &error) { diff --git a/message.cc b/message.cc new file mode 100644 index 00000000..5d99321d --- /dev/null +++ b/message.cc @@ -0,0 +1,131 @@ +/* message.cc - Results of message-based searches from a notmuch database + * + * Copyright © 2009 Carl Worth + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see http://www.gnu.org/licenses/ . + * + * Author: Carl Worth + */ + +#include "notmuch-private.h" +#include "database-private.h" + +#include + +struct _notmuch_message { + Xapian::Document doc; +}; + +struct _notmuch_tags { + Xapian::TermIterator iterator; + Xapian::TermIterator iterator_end; +}; + +static int +_notmuch_message_destroy (notmuch_message_t *message) +{ + message->doc.~Document (); + + return 0; +} + +notmuch_message_t * +_notmuch_message_create (notmuch_results_t *owner, + notmuch_database_t *notmuch, + Xapian::docid doc_id) +{ + notmuch_message_t *message; + + message = talloc (owner, notmuch_message_t); + if (unlikely (message == NULL)) + return NULL; + + new (&message->doc) Xapian::Document; + + talloc_set_destructor (message, _notmuch_message_destroy); + + message->doc = notmuch->xapian_db->get_document (doc_id); + + return message; +} + +const char * +notmuch_message_get_message_id (notmuch_message_t *message) +{ + Xapian::TermIterator i; + + i = message->doc.termlist_begin (); + i.skip_to ("Q"); + if (i != message->doc.termlist_end ()) + return talloc_strdup (message, (*i).c_str () + 1); + else + return NULL; +} + +static int +_notmuch_tags_destroy (notmuch_tags_t *tags) +{ + tags->iterator.~TermIterator (); + tags->iterator_end.~TermIterator (); + + return 0; +} + +notmuch_tags_t * +notmuch_message_get_tags (notmuch_message_t *message) +{ + notmuch_tags_t *tags; + + tags = talloc (message, notmuch_tags_t); + if (unlikely (tags == NULL)) + return NULL; + + new (&tags->iterator) Xapian::TermIterator; + new (&tags->iterator_end) Xapian::TermIterator; + + talloc_set_destructor (tags, _notmuch_tags_destroy); + + tags->iterator = message->doc.termlist_begin (); + tags->iterator.skip_to ("L"); + tags->iterator_end = message->doc.termlist_end (); + + return tags; +} + +notmuch_bool_t +notmuch_tags_has_more (notmuch_tags_t *tags) +{ + std::string s; + + if (tags->iterator == tags->iterator_end) + return FALSE; + + s = *tags->iterator; + if (s.size () && s[0] == 'L') + return TRUE; + else + return FALSE; +} + +const char * +notmuch_tags_get (notmuch_tags_t *tags) +{ + return talloc_strdup (tags, (*tags->iterator).c_str () + 1); +} + +void +notmuch_tags_advance (notmuch_tags_t *tags) +{ + tags->iterator++; +} diff --git a/notmuch-private.h b/notmuch-private.h index 0c20a3c4..728c1a67 100644 --- a/notmuch-private.h +++ b/notmuch-private.h @@ -41,6 +41,31 @@ NOTMUCH_BEGIN_DECLS +#include + + +/* Thanks to Andrew Tridgell's (SAMBA's) talloc for this definition of + * unlikely. The talloc source code comes to us via the GNU LGPL v. 3. + */ +/* these macros gain us a few percent of speed on gcc */ +#if (__GNUC__ >= 3) +/* the strange !! is to ensure that __builtin_expect() takes either 0 or 1 + as its first argument */ +#ifndef likely +#define likely(x) __builtin_expect(!!(x), 1) +#endif +#ifndef unlikely +#define unlikely(x) __builtin_expect(!!(x), 0) +#endif +#else +#ifndef likely +#define likely(x) (x) +#endif +#ifndef unlikely +#define unlikely(x) (x) +#endif +#endif + /* xutil.c */ void * xcalloc (size_t nmemb, size_t size); @@ -57,6 +82,13 @@ xstrdup (const char *s); char * xstrndup (const char *s, size_t n); +/* message.cc */ + +notmuch_message_t * +_notmuch_message_create (notmuch_results_t *owner, + notmuch_database_t *notmuch, + unsigned int doc_id); + /* message-file.c */ /* XXX: I haven't decided yet whether these will actually get exported diff --git a/notmuch.c b/notmuch.c index 1ebd613e..cedfebce 100644 --- a/notmuch.c +++ b/notmuch.c @@ -35,6 +35,8 @@ #include #include +#include + #include /* g_strdup_printf */ #define ARRAY_SIZE(arr) (sizeof (arr) / sizeof (arr[0])) @@ -370,8 +372,76 @@ show_command (int argc, char *argv[]) int dump_command (int argc, char *argv[]) { - fprintf (stderr, "Error: dump is not implemented yet.\n"); - return 1; + FILE *output; + notmuch_database_t *notmuch = NULL; + notmuch_query_t *query; + notmuch_results_t *results; + notmuch_message_t *message; + notmuch_tags_t *tags; + int ret = 0; + + if (argc) { + output = fopen (argv[0], "w"); + if (output == NULL) { + fprintf (stderr, "Error opening %s for writing: %s\n", + argv[1], strerror (errno)); + ret = 1; + goto DONE; + } + } else { + output = stdout; + } + + notmuch = notmuch_database_open (NULL); + if (notmuch == NULL) { + ret = 1; + goto DONE; + } + + query = notmuch_query_create (notmuch, NOTMUCH_QUERY_ALL); + if (query == NULL) { + fprintf (stderr, "Out of memory\n"); + ret = 1; + goto DONE; + } + + notmuch_query_set_sort (query, NOTMUCH_SORT_MESSAGE_ID); + + for (results = notmuch_query_search (query); + notmuch_results_has_more (results); + notmuch_results_advance (results)) + { + message = notmuch_results_get (results); + + fprintf (output, + "%s (", notmuch_message_get_message_id (message)); + + for (tags = notmuch_message_get_tags (message); + notmuch_tags_has_more (tags); + notmuch_tags_advance (tags)) + { + int first = 1; + + if (! first) + fprintf (output, " "); + + fprintf (output, "%s", notmuch_tags_get (tags)); + + first = 0; + } + + fprintf (output, ")\n"); + } + + notmuch_query_destroy (query); + + DONE: + if (notmuch) + notmuch_database_close (notmuch); + if (output != stdout) + fclose (output); + + return ret; } int diff --git a/notmuch.h b/notmuch.h index b63a7d77..6d81fb6c 100644 --- a/notmuch.h +++ b/notmuch.h @@ -31,6 +31,16 @@ NOTMUCH_BEGIN_DECLS +#ifndef FALSE +#define FALSE 0 +#endif + +#ifndef TRUE +#define TRUE 1 +#endif + +typedef int notmuch_bool_t; + /* Status codes used for the return values of most functions. * * A zero value (NOTMUCH_STATUS_SUCCESS) indicates that the function @@ -50,10 +60,13 @@ typedef enum _notmuch_status { NOTMUCH_STATUS_FILE_NOT_EMAIL } notmuch_status_t; -/* An opaque data structure representing a notmuch database. See - * notmuch_database_open and other notmuch_database functions - * below. */ +/* Various opaque data types. For each notmuch__t see the various + * notmuch_ functions below. */ typedef struct _notmuch_database notmuch_database_t; +typedef struct _notmuch_query notmuch_query_t; +typedef struct _notmuch_results notmuch_results_t; +typedef struct _notmuch_message notmuch_message_t; +typedef struct _notmuch_tags notmuch_tags_t; /* Create a new, empty notmuch database located at 'path'. * @@ -150,6 +163,185 @@ notmuch_status_t notmuch_database_add_message (notmuch_database_t *database, const char *filename); +/* Create a new query for 'database'. + * + * Here, 'database' should be an open database, (see + * notmuch_database_open and notmuch_database_create). + * + * For the query string, we'll document the syntax here more + * completely in the future, but it's likely to be a specialized + * version of the general Xapian query syntax: + * + * http://xapian.org/docs/queryparser.html + * + * As a special case, passing a value of NOTMUCH_QUERY_ALL for the + * query string will result in a query that returns all messages in + * the database. + * + * See notmuch_query_set_sort for controlling the order of results and + * notmuch_query_search to actually execute the query. + * + * User should call notmuch_query_destroy when finished with this + * query. + * + * Will return NULL if insufficient memory is available. + */ +notmuch_query_t * +notmuch_query_create (notmuch_database_t *database, + const char *query_string); + +/* Special value to cause notmuch_query_create to return all + * messages. */ +extern const char *NOTMUCH_QUERY_ALL; + +/* Sort values for notmuch_query_set_sort */ +typedef enum { + NOTMUCH_SORT_DATE_OLDEST_FIRST, + NOTMUCH_SORT_DATE_NEWEST_FIRST, + NOTMUCH_SORT_MESSAGE_ID +} notmuch_sort_t; + +/* Specify the sorting desired for this query. */ +void +notmuch_query_set_sort (notmuch_query_t *query, notmuch_sort_t sort); + +/* Execute a query, returning a notmuch_results_t object which can be + * used to iterate over the results. The results object is owned by + * the query and as such, will only be valid until notmuch_query_destroy. + * + * Typical usage might be: + * + * notmuch_query_t *query; + * notmuch_results_t *results; + * + * query = notmuch_query_create (database, query_string); + * + * for (results = notmuch_query_search (query); + * notmuch_results_has_more (results); + * notmuch_result_advance (results)) + * { + * message = notmuch_results_get (results); + * .... + * } + * + * notmuch_query_destroy (query); + * + * Note that there's no explicit destructor for the notmuch_results_t + * object. + */ +notmuch_results_t * +notmuch_query_search (notmuch_query_t *query); + +/* Destroy a notmuch_query_t along with any associated resources. + * + * This will in turn destroy any notmuch_results_t objects generated + * by this query, (and in turn any notmuch_message_t objects generated + * from those results, etc.). + */ +void +notmuch_query_destroy (notmuch_query_t *query); + +/* Does the given notmuch_results_t object contain any more results. + * + * When this function returns TRUE, notmuch_results_get will return a + * valid object. Whereas when this function returns FALSE, + * notmuch_results_get will return NULL. + * + * See the documentation of notmuch_query_search for example code + * showing how to iterate over a notmuch_results_t object. + */ +notmuch_bool_t +notmuch_results_has_more (notmuch_results_t *results); + +/* Get the current result from 'results' as a notmuch_message_t. + * + * Note: The returned message belongs to 'results' and has a lifetime + * identical to it (and the query to which it belongs). + * + * See the documentation of notmuch_query_search for example code + * showing how to iterate over a notmuch_results_t object. + */ +notmuch_message_t * +notmuch_results_get (notmuch_results_t *results); + +/* Advance the 'results' iterator to the next result. + * + * See the documentation of notmuch_query_search for example code + * showing how to iterate over a notmuch_results_t object. + */ +void +notmuch_results_advance (notmuch_results_t *results); + +/* Get the message ID of 'message'. + * + * The returned string belongs to 'message' and as such, should not be + * modified by the caller and will only be valid for as long as the + * message is valid, (which is until the query from which it derived + * is destroyed). + */ +const char * +notmuch_message_get_message_id (notmuch_message_t *message); + +/* Get the tags for 'message', returning a notmuch_tags_t object which + * can be used to iterate over all tags. + * + * The tags object is owned by the message and as such, will only be + * valid for as long as the message is valid, (which is until the + * query from which it derived is destroyed). + * + * Typical usage might be: + * + * notmuch_message_t *message; + * notmuch_tags_t *tags; + * const char *tag; + * + * message = notmuch_results_get (results); + * + * for (tags = notmuch_message_get_tags (message); + * notmuch_tags_has_more (tags); + * notmuch_result_advance (tags)) + * { + * tag = notmuch_tags_get_string (tags); + * .... + * } + * + * Note that there's no explicit destructor for the notmuch_tags_t + * object. + */ +notmuch_tags_t * +notmuch_message_get_tags (notmuch_message_t *message); + +/* Does the given notmuch_tags_t object contain any more results. + * + * When this function returns TRUE, notmuch_tags_get will return a + * valid string. Whereas when this function returns FALSE, + * notmuch_tags_get will return NULL. + * + * See the documentation of notmuch_message_get_tags for example code + * showing how to iterate over a notmuch_tags_t object. + */ +notmuch_bool_t +notmuch_tags_has_more (notmuch_tags_t *tags); + +/* Get the current result from 'tags' as a string. + * + * Note: The returned string belongs to 'tags' and has a lifetime + * identical to it (and the query to which it utlimately belongs). + * + * See the documentation of notmuch_message_get_tags for example code + * showing how to iterate over a notmuch_tags_t object. + */ +const char * +notmuch_tags_get (notmuch_tags_t *tags); + +/* Advance the 'tags' iterator to the next tag. + * + * See the documentation of notmuch_message_get_tags for example code + * showing how to iterate over a notmuch_tags_t object. + */ +void +notmuch_tags_advance (notmuch_tags_t *results); + NOTMUCH_END_DECLS #endif diff --git a/query.cc b/query.cc new file mode 100644 index 00000000..d3614dc3 --- /dev/null +++ b/query.cc @@ -0,0 +1,137 @@ +/* query.cc - Support for searching a notmuch database + * + * Copyright © 2009 Carl Worth + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see http://www.gnu.org/licenses/ . + * + * Author: Carl Worth + */ + +#include "notmuch-private.h" +#include "database-private.h" + +#include + +const char *NOTMUCH_QUERY_ALL = ""; + +struct _notmuch_query { + notmuch_database_t *notmuch; + const char *query_string; + notmuch_sort_t sort; +}; + +struct _notmuch_results { + notmuch_database_t *notmuch; + Xapian::PostingIterator iterator; + Xapian::PostingIterator iterator_end; +}; + +notmuch_query_t * +notmuch_query_create (notmuch_database_t *notmuch, + const char *query_string) +{ + notmuch_query_t *query; + + query = talloc (NULL, notmuch_query_t); + if (unlikely (query == NULL)) + return NULL; + + query->notmuch = notmuch; + + /* Special-case NOTMUCH_QUERY_ALL so we see it and not a copy. */ + if (query_string == NOTMUCH_QUERY_ALL) + query->query_string = query_string; + else + query->query_string = talloc_strdup (query, query_string); + + query->sort = NOTMUCH_SORT_DATE_OLDEST_FIRST; + + return query; +} + +void +notmuch_query_set_sort (notmuch_query_t *query, notmuch_sort_t sort) +{ + query->sort = sort; +} + +static int +_notmuch_results_destroy (notmuch_results_t *results) +{ + results->iterator.~PostingIterator (); + results->iterator_end.~PostingIterator (); + + return 0; +} + +notmuch_results_t * +notmuch_query_search (notmuch_query_t *query) +{ + notmuch_results_t *results; + + results = talloc (query, notmuch_results_t); + if (unlikely (results == NULL)) + return NULL; + + try { + if (query->query_string != NOTMUCH_QUERY_ALL) { + fprintf (stderr, "Error: Arbitrary search strings are not supported yet. Come back soon!\n"); + exit (1); + } + + results->notmuch = query->notmuch; + new (&results->iterator) Xapian::PostingIterator (); + new (&results->iterator_end) Xapian::PostingIterator (); + + talloc_set_destructor (results, _notmuch_results_destroy); + + results->iterator = query->notmuch->xapian_db->postlist_begin (""); + results->iterator_end = query->notmuch->xapian_db->postlist_end (""); + + } catch (const Xapian::Error &error) { + fprintf (stderr, "A Xapian exception occurred: %s\n", + error.get_msg().c_str()); + } + + return results; +} + +void +notmuch_query_destroy (notmuch_query_t *query) +{ + talloc_free (query); +} + +notmuch_bool_t +notmuch_results_has_more (notmuch_results_t *results) +{ + return (results->iterator != results->iterator_end); +} + +notmuch_message_t * +notmuch_results_get (notmuch_results_t *results) +{ + Xapian::docid doc_id; + + doc_id = *results->iterator; + + return _notmuch_message_create (results, + results->notmuch, doc_id); +} + +void +notmuch_results_advance (notmuch_results_t *results) +{ + results->iterator++; +} -- 2.43.0 From f6c7810945f1bc25b15dee72257c3b68bd0e8a40 Mon Sep 17 00:00:00 2001 From: Carl Worth Date: Tue, 20 Oct 2009 22:08:31 -0700 Subject: [PATCH 08/16] Rename our talloc destructor functions to _destructor. I want to reserve the _destroy names for some public functions I'm about to add. --- message.cc | 20 ++++++++++++++++---- query.cc | 10 ++++++++-- 2 files changed, 24 insertions(+), 6 deletions(-) diff --git a/message.cc b/message.cc index 5d99321d..cc157c8b 100644 --- a/message.cc +++ b/message.cc @@ -32,8 +32,14 @@ struct _notmuch_tags { Xapian::TermIterator iterator_end; }; +/* We end up having to call the destructor explicitly because we had + * to use "placement new" in order to initialize C++ objects within a + * block that we allocated with talloc. So C++ is making talloc + * slightly less simple to use, (we wouldn't need + * talloc_set_destructor at all otherwise). + */ static int -_notmuch_message_destroy (notmuch_message_t *message) +_notmuch_message_destructor (notmuch_message_t *message) { message->doc.~Document (); @@ -53,7 +59,7 @@ _notmuch_message_create (notmuch_results_t *owner, new (&message->doc) Xapian::Document; - talloc_set_destructor (message, _notmuch_message_destroy); + talloc_set_destructor (message, _notmuch_message_destructor); message->doc = notmuch->xapian_db->get_document (doc_id); @@ -73,8 +79,14 @@ notmuch_message_get_message_id (notmuch_message_t *message) return NULL; } +/* We end up having to call the destructors explicitly because we had + * to use "placement new" in order to initialize C++ objects within a + * block that we allocated with talloc. So C++ is making talloc + * slightly less simple to use, (we wouldn't need + * talloc_set_destructor at all otherwise). + */ static int -_notmuch_tags_destroy (notmuch_tags_t *tags) +_notmuch_tags_destructor (notmuch_tags_t *tags) { tags->iterator.~TermIterator (); tags->iterator_end.~TermIterator (); @@ -94,7 +106,7 @@ notmuch_message_get_tags (notmuch_message_t *message) new (&tags->iterator) Xapian::TermIterator; new (&tags->iterator_end) Xapian::TermIterator; - talloc_set_destructor (tags, _notmuch_tags_destroy); + talloc_set_destructor (tags, _notmuch_tags_destructor); tags->iterator = message->doc.termlist_begin (); tags->iterator.skip_to ("L"); diff --git a/query.cc b/query.cc index d3614dc3..c669fb91 100644 --- a/query.cc +++ b/query.cc @@ -66,8 +66,14 @@ notmuch_query_set_sort (notmuch_query_t *query, notmuch_sort_t sort) query->sort = sort; } +/* We end up having to call the destructors explicitly because we had + * to use "placement new" in order to initialize C++ objects within a + * block that we allocated with talloc. So C++ is making talloc + * slightly less simple to use, (we wouldn't need + * talloc_set_destructor at all otherwise). + */ static int -_notmuch_results_destroy (notmuch_results_t *results) +_notmuch_results_destructor (notmuch_results_t *results) { results->iterator.~PostingIterator (); results->iterator_end.~PostingIterator (); @@ -94,7 +100,7 @@ notmuch_query_search (notmuch_query_t *query) new (&results->iterator) Xapian::PostingIterator (); new (&results->iterator_end) Xapian::PostingIterator (); - talloc_set_destructor (results, _notmuch_results_destroy); + talloc_set_destructor (results, _notmuch_results_destructor); results->iterator = query->notmuch->xapian_db->postlist_begin (""); results->iterator_end = query->notmuch->xapian_db->postlist_end (""); -- 2.43.0 From 4ca1492f1b6a9172b1dca88aecf1d6e7394ac5d7 Mon Sep 17 00:00:00 2001 From: Carl Worth Date: Tue, 20 Oct 2009 22:24:59 -0700 Subject: [PATCH 09/16] Add destroy functions for results, message, and tags. None of these are strictly necessary, (everything was leak-free without them), but notmuch_message_destroy can actually be useful for when one query has many message results, but only one is needed to be live at a time. The destroy functions for results and tags are fairly gratuitous, as there's unlikely to be any benefit from calling them. But they're all easy to add, (all of these functions are just wrappers for talloc_free), and we do so for consistency and completeness. --- message.cc | 12 ++++++++++++ notmuch.h | 44 ++++++++++++++++++++++++++++++++++++++++---- query.cc | 6 ++++++ 3 files changed, 58 insertions(+), 4 deletions(-) diff --git a/message.cc b/message.cc index cc157c8b..ca4a16c7 100644 --- a/message.cc +++ b/message.cc @@ -115,6 +115,12 @@ notmuch_message_get_tags (notmuch_message_t *message) return tags; } +void +notmuch_message_destroy (notmuch_message_t *message) +{ + talloc_free (message); +} + notmuch_bool_t notmuch_tags_has_more (notmuch_tags_t *tags) { @@ -141,3 +147,9 @@ notmuch_tags_advance (notmuch_tags_t *tags) { tags->iterator++; } + +void +notmuch_tags_destroy (notmuch_tags_t *tags) +{ + talloc_free (tags); +} diff --git a/notmuch.h b/notmuch.h index 6d81fb6c..10067d3f 100644 --- a/notmuch.h +++ b/notmuch.h @@ -226,8 +226,13 @@ notmuch_query_set_sort (notmuch_query_t *query, notmuch_sort_t sort); * * notmuch_query_destroy (query); * - * Note that there's no explicit destructor for the notmuch_results_t - * object. + * Note that there's no explicit destructor needed for the + * notmuch_results_t object. + * + * (For consistency, we do provide a notmuch_results_destroy function, + * but there's no point in calling it if you're about to destroy the + * query object as well too---either call will free all the memory of + * the results). */ notmuch_results_t * notmuch_query_search (notmuch_query_t *query); @@ -272,6 +277,15 @@ notmuch_results_get (notmuch_results_t *results); void notmuch_results_advance (notmuch_results_t *results); +/* Destroy a notmuch_results_t object. + * + * It's not strictly necessary to call this function. All memory from + * the notmuch_results_t object will be reclaimed when the containg + * query object is destroyed. + */ +void +notmuch_results_destroy (notmuch_results_t *results); + /* Get the message ID of 'message'. * * The returned string belongs to 'message' and as such, should not be @@ -305,12 +319,25 @@ notmuch_message_get_message_id (notmuch_message_t *message); * .... * } * - * Note that there's no explicit destructor for the notmuch_tags_t - * object. + * Note: If you are finished with a message before its containing + * query, you can call notmuch_message_destroy to clean up some memory + * sooner. If you don't call it, all the memory will still be + * reclaimed when the query is destroyed. */ notmuch_tags_t * notmuch_message_get_tags (notmuch_message_t *message); +/* Destroy a notmuch_message_t object. + * + * It can be useful to call this function in the case of a single + * query object with many messages in the result, (such as iterating + * over the entire database). Otherwise, it's fine to never call this + * function and there will still be no memory leaks. (The memory from + * the messages get reclaimed when the containing query is destroyed.) + */ +void +notmuch_message_destroy (notmuch_message_t *message); + /* Does the given notmuch_tags_t object contain any more results. * * When this function returns TRUE, notmuch_tags_get will return a @@ -342,6 +369,15 @@ notmuch_tags_get (notmuch_tags_t *tags); void notmuch_tags_advance (notmuch_tags_t *results); +/* Destroy a notmuch_tags_t object. + * + * It's not strictly necessary to call this function. All memory from + * the notmuch_tags_t object will be reclaimed when the containg + * message or query objects are destroyed. + */ +void +notmuch_tags_destroy (notmuch_tags_t *tags); + NOTMUCH_END_DECLS #endif diff --git a/query.cc b/query.cc index c669fb91..2a1815a7 100644 --- a/query.cc +++ b/query.cc @@ -141,3 +141,9 @@ notmuch_results_advance (notmuch_results_t *results) { results->iterator++; } + +void +notmuch_results_destroy (notmuch_results_t *results) +{ + talloc_free (results); +} -- 2.43.0 From aad13c3ac947a42b3e0a1923c44a927feea827f5 Mon Sep 17 00:00:00 2001 From: Carl Worth Date: Tue, 20 Oct 2009 22:27:56 -0700 Subject: [PATCH 10/16] notmuch dump: Free each message as it's used. Previously we were leaking[*] memory in that the memory footprint of a "notmuch dump" run would continue to grow until the output was complete, and then finally all the memory would be freed. Now, the memory footprint is small and constant, O(1) rather than O(n) in the number of messages. [*] Not leaking in a valgrind sense---every byte was still carefully being accounted for and freed eventually. --- notmuch.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/notmuch.c b/notmuch.c index cedfebce..43af75d2 100644 --- a/notmuch.c +++ b/notmuch.c @@ -431,6 +431,8 @@ dump_command (int argc, char *argv[]) } fprintf (output, ")\n"); + + notmuch_message_destroy (message); } notmuch_query_destroy (query); -- 2.43.0 From 6519aff957df7a69d86ed9a00707c911d60259f6 Mon Sep 17 00:00:00 2001 From: Carl Worth Date: Tue, 20 Oct 2009 22:40:37 -0700 Subject: [PATCH 11/16] query: Remove the magic NOTMUCH_QUERY_ALL Using the address of a static char* was clever, but really unnecessary. An empty string is much less magic, and even easier to understand as the way to query everything from the database. --- notmuch.c | 2 +- notmuch.h | 9 ++------- query.cc | 10 ++-------- 3 files changed, 5 insertions(+), 16 deletions(-) diff --git a/notmuch.c b/notmuch.c index 43af75d2..6be2881c 100644 --- a/notmuch.c +++ b/notmuch.c @@ -398,7 +398,7 @@ dump_command (int argc, char *argv[]) goto DONE; } - query = notmuch_query_create (notmuch, NOTMUCH_QUERY_ALL); + query = notmuch_query_create (notmuch, ""); if (query == NULL) { fprintf (stderr, "Out of memory\n"); ret = 1; diff --git a/notmuch.h b/notmuch.h index 10067d3f..df432bcc 100644 --- a/notmuch.h +++ b/notmuch.h @@ -174,9 +174,8 @@ notmuch_database_add_message (notmuch_database_t *database, * * http://xapian.org/docs/queryparser.html * - * As a special case, passing a value of NOTMUCH_QUERY_ALL for the - * query string will result in a query that returns all messages in - * the database. + * As a special case, passing a length-zero string, (that is ""), will + * result in a query that returns all messages in the database. * * See notmuch_query_set_sort for controlling the order of results and * notmuch_query_search to actually execute the query. @@ -190,10 +189,6 @@ notmuch_query_t * notmuch_query_create (notmuch_database_t *database, const char *query_string); -/* Special value to cause notmuch_query_create to return all - * messages. */ -extern const char *NOTMUCH_QUERY_ALL; - /* Sort values for notmuch_query_set_sort */ typedef enum { NOTMUCH_SORT_DATE_OLDEST_FIRST, diff --git a/query.cc b/query.cc index 2a1815a7..50223b02 100644 --- a/query.cc +++ b/query.cc @@ -23,8 +23,6 @@ #include -const char *NOTMUCH_QUERY_ALL = ""; - struct _notmuch_query { notmuch_database_t *notmuch; const char *query_string; @@ -49,11 +47,7 @@ notmuch_query_create (notmuch_database_t *notmuch, query->notmuch = notmuch; - /* Special-case NOTMUCH_QUERY_ALL so we see it and not a copy. */ - if (query_string == NOTMUCH_QUERY_ALL) - query->query_string = query_string; - else - query->query_string = talloc_strdup (query, query_string); + query->query_string = talloc_strdup (query, query_string); query->sort = NOTMUCH_SORT_DATE_OLDEST_FIRST; @@ -91,7 +85,7 @@ notmuch_query_search (notmuch_query_t *query) return NULL; try { - if (query->query_string != NOTMUCH_QUERY_ALL) { + if (strlen (query->query_string)) { fprintf (stderr, "Error: Arbitrary search strings are not supported yet. Come back soon!\n"); exit (1); } -- 2.43.0 From 50144fb354dc5bf282c2a2cdc68c926e42ccf3ef Mon Sep 17 00:00:00 2001 From: Carl Worth Date: Tue, 20 Oct 2009 23:12:53 -0700 Subject: [PATCH 12/16] database: Remove two little bits of dead code. --- database.cc | 3 --- 1 file changed, 3 deletions(-) diff --git a/database.cc b/database.cc index 041cffdc..5ee07059 100644 --- a/database.cc +++ b/database.cc @@ -457,7 +457,6 @@ notmuch_database_open (const char *path) notmuch->path = xstrdup (path); try { - Xapian::PostingIterator i; notmuch->xapian_db = new Xapian::WritableDatabase (xapian_path, Xapian::DB_CREATE_OR_OPEN); } catch (const Xapian::Error &error) { @@ -519,8 +518,6 @@ notmuch_database_add_message (notmuch_database_t *notmuch, (char *) NULL); try { - doc = Xapian::Document (); - doc.set_data (filename); parents = g_ptr_array_new (); -- 2.43.0 From 266c612a5035ab0494c25e40f55aef097e6c99ce Mon Sep 17 00:00:00 2001 From: Carl Worth Date: Tue, 20 Oct 2009 23:13:28 -0700 Subject: [PATCH 13/16] .gitignore: Ignore generated file Makefile.dep Forgot to add this when I first add dependency checking to the Makefile. --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 792e79ca..ad5e336b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ +Makefile.dep notmuch -- 2.43.0 From af65f52acf4eac5b2187855a63895afe3386c074 Mon Sep 17 00:00:00 2001 From: Carl Worth Date: Wed, 21 Oct 2009 00:32:30 -0700 Subject: [PATCH 14/16] notmuch setup: Print a few protecting spaces after progress reports. This is to help keep the report looking clean when a new report is shorter than a previous reports, (say, when crossing the boundary from over one minute remaining to less than one minute remaining). This used to be here, but I must have accidentally dropped it when reformatting the progress report recently. --- notmuch.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/notmuch.c b/notmuch.c index 6be2881c..a93df823 100644 --- a/notmuch.c +++ b/notmuch.c @@ -99,7 +99,7 @@ add_files_print_progress (add_files_state_t *state) state->count, state->total_messages); print_formatted_seconds ((state->total_messages - state->count) / rate_overall); - printf (" remaining).\r"); + printf (" remaining). \r"); fflush (stdout); } -- 2.43.0 From 6a3b68edeffa53c3e1c9aa156eff46c5999077c5 Mon Sep 17 00:00:00 2001 From: Carl Worth Date: Wed, 21 Oct 2009 00:34:36 -0700 Subject: [PATCH 15/16] add_message: Add a type:mail ("Kmail") term to all documents. This gives us an easy way to specify "all mail messages" in a search query. We simply look for this term. --- database.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/database.cc b/database.cc index 5ee07059..1c1e590b 100644 --- a/database.cc +++ b/database.cc @@ -520,6 +520,8 @@ notmuch_database_add_message (notmuch_database_t *notmuch, try { doc.set_data (filename); + add_term (doc, "type", "mail"); + parents = g_ptr_array_new (); refs = notmuch_message_file_get_header (message, "references"); -- 2.43.0 From 65baa4f4e7fc401e5af742b491a3bc0784f2cdf7 Mon Sep 17 00:00:00 2001 From: Carl Worth Date: Wed, 21 Oct 2009 00:35:56 -0700 Subject: [PATCH 16/16] notmuch dump: Fix the sorting of results. To properly support sorting in notmuch_query we know use an Enquire object. We also throw in a QueryParser too, so we're really close to being able to support arbitrary full-text searches. I took a look at the supported QueryParser syntax and chose a set of flags for everything I like, (such as supporting Boolean operators in either case ("AND" or "and"), supporting phrase searching, supporting + and - to include/preclude terms, and supporting a trailing * on any term as a wildcard). --- database-private.h | 2 +- database.cc | 13 ++++------- notmuch-private.h | 9 ++++++++ query.cc | 57 ++++++++++++++++++++++++++++++++++++---------- 4 files changed, 59 insertions(+), 22 deletions(-) diff --git a/database-private.h b/database-private.h index b894717e..a5cca5a4 100644 --- a/database-private.h +++ b/database-private.h @@ -28,7 +28,7 @@ struct _notmuch_database { char *path; Xapian::WritableDatabase *xapian_db; - Xapian::TermGenerator *term_gen; + Xapian::QueryParser *query_parser; }; #endif diff --git a/database.cc b/database.cc index 1c1e590b..31afe7cc 100644 --- a/database.cc +++ b/database.cc @@ -67,15 +67,6 @@ prefix_t BOOLEAN_PREFIX[] = { { "ref", "R" } }; -/* Similarly, these value numbers are also chosen to be sup - * compatible. */ - -typedef enum { - NOTMUCH_VALUE_MESSAGE_ID = 0, - NOTMUCH_VALUE_THREAD = 1, - NOTMUCH_VALUE_DATE = 2 -} notmuch_value_t; - static const char * find_prefix (const char *name) { @@ -459,6 +450,9 @@ notmuch_database_open (const char *path) try { notmuch->xapian_db = new Xapian::WritableDatabase (xapian_path, Xapian::DB_CREATE_OR_OPEN); + notmuch->query_parser = new Xapian::QueryParser; + notmuch->query_parser->set_default_op (Xapian::Query::OP_AND); + notmuch->query_parser->set_database (*notmuch->xapian_db); } catch (const Xapian::Error &error) { fprintf (stderr, "A Xapian exception occurred: %s\n", error.get_msg().c_str()); @@ -478,6 +472,7 @@ notmuch_database_open (const char *path) void notmuch_database_close (notmuch_database_t *notmuch) { + delete notmuch->query_parser; delete notmuch->xapian_db; free (notmuch->path); free (notmuch); diff --git a/notmuch-private.h b/notmuch-private.h index 728c1a67..2bf6beb4 100644 --- a/notmuch-private.h +++ b/notmuch-private.h @@ -66,6 +66,15 @@ NOTMUCH_BEGIN_DECLS #endif #endif +/* These value numbers are chosen to be sup compatible (for now at + * least). */ + +typedef enum { + NOTMUCH_VALUE_MESSAGE_ID = 0, + NOTMUCH_VALUE_THREAD = 1, + NOTMUCH_VALUE_DATE = 2 +} notmuch_value_t; + /* xutil.c */ void * xcalloc (size_t nmemb, size_t size); diff --git a/query.cc b/query.cc index 50223b02..a15de966 100644 --- a/query.cc +++ b/query.cc @@ -31,8 +31,8 @@ struct _notmuch_query { struct _notmuch_results { notmuch_database_t *notmuch; - Xapian::PostingIterator iterator; - Xapian::PostingIterator iterator_end; + Xapian::MSetIterator iterator; + Xapian::MSetIterator iterator_end; }; notmuch_query_t * @@ -69,8 +69,8 @@ notmuch_query_set_sort (notmuch_query_t *query, notmuch_sort_t sort) static int _notmuch_results_destructor (notmuch_results_t *results) { - results->iterator.~PostingIterator (); - results->iterator_end.~PostingIterator (); + results->iterator.~MSetIterator (); + results->iterator_end.~MSetIterator (); return 0; } @@ -78,6 +78,8 @@ _notmuch_results_destructor (notmuch_results_t *results) notmuch_results_t * notmuch_query_search (notmuch_query_t *query) { + notmuch_database_t *notmuch = query->notmuch; + const char *query_string = query->query_string; notmuch_results_t *results; results = talloc (query, notmuch_results_t); @@ -85,19 +87,50 @@ notmuch_query_search (notmuch_query_t *query) return NULL; try { - if (strlen (query->query_string)) { - fprintf (stderr, "Error: Arbitrary search strings are not supported yet. Come back soon!\n"); - exit (1); + Xapian::Enquire enquire (*notmuch->xapian_db); + Xapian::Query mail_query ("Kmail"); + Xapian::Query string_query, final_query; + Xapian::MSet mset; + unsigned int flags = (Xapian::QueryParser::FLAG_BOOLEAN & + Xapian::QueryParser::FLAG_PHRASE & + Xapian::QueryParser::FLAG_LOVEHATE & + Xapian::QueryParser::FLAG_BOOLEAN_ANY_CASE & + Xapian::QueryParser::FLAG_WILDCARD); + + if (strcmp (query_string, "") == 0) { + final_query = mail_query; + } else { + string_query = notmuch->query_parser-> + parse_query (query_string, flags); + final_query = Xapian::Query (Xapian::Query::OP_AND, + mail_query, string_query); } - results->notmuch = query->notmuch; - new (&results->iterator) Xapian::PostingIterator (); - new (&results->iterator_end) Xapian::PostingIterator (); + switch (query->sort) { + case NOTMUCH_SORT_DATE_OLDEST_FIRST: + enquire.set_sort_by_value (NOTMUCH_VALUE_DATE, FALSE); + break; + case NOTMUCH_SORT_DATE_NEWEST_FIRST: + enquire.set_sort_by_value (NOTMUCH_VALUE_DATE, TRUE); + break; + case NOTMUCH_SORT_MESSAGE_ID: + enquire.set_sort_by_value (NOTMUCH_VALUE_MESSAGE_ID, FALSE); + break; + } + + enquire.set_query (final_query); + + mset = enquire.get_mset (0, notmuch->xapian_db->get_doccount ()); + + results->notmuch = notmuch; + + new (&results->iterator) Xapian::MSetIterator (); + new (&results->iterator_end) Xapian::MSetIterator (); talloc_set_destructor (results, _notmuch_results_destructor); - results->iterator = query->notmuch->xapian_db->postlist_begin (""); - results->iterator_end = query->notmuch->xapian_db->postlist_end (""); + results->iterator = mset.begin (); + results->iterator_end = mset.end (); } catch (const Xapian::Error &error) { fprintf (stderr, "A Xapian exception occurred: %s\n", -- 2.43.0