]> git.notmuchmail.org Git - notmuch/blobdiff - lib/query.cc
lib: work around xapian bug with get_mset(0,0, x)
[notmuch] / lib / query.cc
index 53efd4e18fc1698d88c0b8ac476fd33094f2e2fc..7fdf992de7162a758d6ea869515ad07e15aaa4a6 100644 (file)
@@ -29,6 +29,9 @@ struct _notmuch_query {
     notmuch_sort_t sort;
     notmuch_string_list_t *exclude_terms;
     notmuch_exclude_t omit_excluded;
+    bool parsed;
+    Xapian::Query xapian_query;
+    std::set<std::string> terms;
 };
 
 typedef struct _notmuch_mset_messages {
@@ -46,7 +49,7 @@ struct _notmuch_doc_id_set {
 #define DOCIDSET_WORD(bit) ((bit) / CHAR_BIT)
 #define DOCIDSET_BIT(bit) ((bit) % CHAR_BIT)
 
-struct visible _notmuch_threads {
+struct _notmuch_threads {
     notmuch_query_t *query;
 
     /* The ordered list of doc ids matched by the query. */
@@ -59,18 +62,26 @@ struct visible _notmuch_threads {
 };
 
 /* We need this in the message functions so forward declare. */
-static notmuch_bool_t
+static bool
 _notmuch_doc_id_set_init (void *ctx,
                          notmuch_doc_id_set_t *doc_ids,
                          GArray *arr);
 
-static notmuch_bool_t
+static bool
 _debug_query (void)
 {
     char *env = getenv ("NOTMUCH_DEBUG_QUERY");
     return (env && strcmp (env, "") != 0);
 }
 
+/* Explicit destructor call for placement new */
+static int
+_notmuch_query_destructor (notmuch_query_t *query) {
+    query->xapian_query.~Query();
+    query->terms.~set<std::string>();
+    return 0;
+}
+
 notmuch_query_t *
 notmuch_query_create (notmuch_database_t *notmuch,
                      const char *query_string)
@@ -84,6 +95,12 @@ notmuch_query_create (notmuch_database_t *notmuch,
     if (unlikely (query == NULL))
        return NULL;
 
+    new (&query->xapian_query) Xapian::Query ();
+    new (&query->terms) std::set<std::string> ();
+    query->parsed = false;
+
+    talloc_set_destructor (query, _notmuch_query_destructor);
+
     query->notmuch = notmuch;
 
     query->query_string = talloc_strdup (query, query_string);
@@ -97,6 +114,44 @@ notmuch_query_create (notmuch_database_t *notmuch,
     return query;
 }
 
+static notmuch_status_t
+_notmuch_query_ensure_parsed (notmuch_query_t *query)
+{
+    if (query->parsed)
+       return NOTMUCH_STATUS_SUCCESS;
+
+    try {
+       query->xapian_query =
+           query->notmuch->query_parser->
+               parse_query (query->query_string, NOTMUCH_QUERY_PARSER_FLAGS);
+
+       /* Xapian doesn't support skip_to on terms from a query since
+       *  they are unordered, so cache a copy of all terms in
+       *  something searchable.
+       */
+
+       for (Xapian::TermIterator t = query->xapian_query.get_terms_begin ();
+            t != query->xapian_query.get_terms_end (); ++t)
+           query->terms.insert (*t);
+
+       query->parsed = true;
+
+    } catch (const Xapian::Error &error) {
+       if (!query->notmuch->exception_reported) {
+           _notmuch_database_log (query->notmuch,
+                                  "A Xapian exception occurred parsing query: %s\n",
+                                  error.get_msg ().c_str ());
+           _notmuch_database_log_append (query->notmuch,
+                                         "Query string was: %s\n",
+                                         query->query_string);
+           query->notmuch->exception_reported = true;
+       }
+
+       return NOTMUCH_STATUS_XAPIAN_EXCEPTION;
+    }
+    return NOTMUCH_STATUS_SUCCESS;
+}
+
 const char *
 notmuch_query_get_query_string (const notmuch_query_t *query)
 {
@@ -122,11 +177,22 @@ notmuch_query_get_sort (const notmuch_query_t *query)
     return query->sort;
 }
 
-void
+notmuch_status_t
 notmuch_query_add_tag_exclude (notmuch_query_t *query, const char *tag)
 {
-    char *term = talloc_asprintf (query, "%s%s", _find_prefix ("tag"), tag);
+    notmuch_status_t status;
+    char *term;
+
+    status = _notmuch_query_ensure_parsed (query);
+    if (status)
+       return status;
+
+    term = talloc_asprintf (query, "%s%s", _find_prefix ("tag"), tag);
+    if (query->terms.count(term) != 0)
+       return NOTMUCH_STATUS_IGNORED;
+
     _notmuch_string_list_append (query->exclude_terms, term);
+    return NOTMUCH_STATUS_SUCCESS;
 }
 
 /* We end up having to call the destructors explicitly because we had
@@ -145,46 +211,31 @@ _notmuch_messages_destructor (notmuch_mset_messages_t *messages)
 }
 
 /* Return a query that matches messages with the excluded tags
- * registered with query.  Any tags that explicitly appear in xquery
- * will not be excluded, and will be removed from the list of exclude
- * tags.  The caller of this function has to combine the returned
+ * registered with query. The caller of this function has to combine the returned
  * query appropriately.*/
 static Xapian::Query
-_notmuch_exclude_tags (notmuch_query_t *query, Xapian::Query xquery)
+_notmuch_exclude_tags (notmuch_query_t *query)
 {
     Xapian::Query exclude_query = Xapian::Query::MatchNothing;
 
     for (notmuch_string_node_t *term = query->exclude_terms->head; term;
         term = term->next) {
-       Xapian::TermIterator it = xquery.get_terms_begin ();
-       Xapian::TermIterator end = xquery.get_terms_end ();
-       for (; it != end; it++) {
-           if ((*it).compare (term->string) == 0)
-               break;
-       }
-       if (it == end)
-           exclude_query = Xapian::Query (Xapian::Query::OP_OR,
-                                   exclude_query, Xapian::Query (term->string));
-       else
-           term->string = talloc_strdup (query, "");
+       exclude_query = Xapian::Query (Xapian::Query::OP_OR,
+                                      exclude_query, Xapian::Query (term->string));
     }
     return exclude_query;
 }
 
-notmuch_messages_t *
-notmuch_query_search_messages (notmuch_query_t *query)
+
+notmuch_status_t
+notmuch_query_search_messages_st (notmuch_query_t *query,
+                                 notmuch_messages_t **out)
 {
-    notmuch_status_t status;
-    notmuch_messages_t *messages;
-    status = notmuch_query_search_messages_st (query, &messages);
-    if (status)
-       return NULL;
-    else
-       return messages;
+    return notmuch_query_search_messages (query, out);
 }
 
 notmuch_status_t
-notmuch_query_search_messages_st (notmuch_query_t *query,
+notmuch_query_search_messages (notmuch_query_t *query,
                                  notmuch_messages_t **out)
 {
     return _notmuch_query_search_documents (query, "mail", out);
@@ -198,6 +249,11 @@ _notmuch_query_search_documents (notmuch_query_t *query,
     notmuch_database_t *notmuch = query->notmuch;
     const char *query_string = query->query_string;
     notmuch_mset_messages_t *messages;
+    notmuch_status_t status;
+
+    status = _notmuch_query_ensure_parsed (query);
+    if (status)
+       return status;
 
     messages = talloc (query, notmuch_mset_messages_t);
     if (unlikely (messages == NULL))
@@ -205,7 +261,7 @@ _notmuch_query_search_documents (notmuch_query_t *query,
 
     try {
 
-       messages->base.is_of_list_type = FALSE;
+       messages->base.is_of_list_type = false;
        messages->base.iterator = NULL;
        messages->notmuch = notmuch;
        new (&messages->iterator) Xapian::MSetIterator ();
@@ -217,7 +273,7 @@ _notmuch_query_search_documents (notmuch_query_t *query,
        Xapian::Query mail_query (talloc_asprintf (query, "%s%s",
                                                   _find_prefix ("type"),
                                                   type));
-       Xapian::Query string_query, final_query, exclude_query;
+       Xapian::Query final_query, exclude_query;
        Xapian::MSet mset;
        Xapian::MSetIterator iterator;
 
@@ -226,15 +282,13 @@ _notmuch_query_search_documents (notmuch_query_t *query,
        {
            final_query = mail_query;
        } else {
-           string_query = notmuch->query_parser->
-               parse_query (query_string, NOTMUCH_QUERY_PARSER_FLAGS);
            final_query = Xapian::Query (Xapian::Query::OP_AND,
-                                        mail_query, string_query);
+                                        mail_query, query->xapian_query);
        }
        messages->base.excluded_doc_ids = NULL;
 
        if ((query->omit_excluded != NOTMUCH_EXCLUDE_FALSE) && (query->exclude_terms)) {
-           exclude_query = _notmuch_exclude_tags (query, final_query);
+           exclude_query = _notmuch_exclude_tags (query);
 
            if (query->omit_excluded == NOTMUCH_EXCLUDE_TRUE ||
                query->omit_excluded == NOTMUCH_EXCLUDE_ALL)
@@ -250,7 +304,7 @@ _notmuch_query_search_documents (notmuch_query_t *query,
 
                mset = enquire.get_mset (0, notmuch->xapian_db->get_doccount ());
 
-               GArray *excluded_doc_ids = g_array_new (FALSE, FALSE, sizeof (unsigned int));
+               GArray *excluded_doc_ids = g_array_new (false, false, sizeof (unsigned int));
 
                for (iterator = mset.begin (); iterator != mset.end (); iterator++) {
                    unsigned int doc_id = *iterator;
@@ -268,13 +322,13 @@ _notmuch_query_search_documents (notmuch_query_t *query,
 
        switch (query->sort) {
        case NOTMUCH_SORT_OLDEST_FIRST:
-           enquire.set_sort_by_value (NOTMUCH_VALUE_TIMESTAMP, FALSE);
+           enquire.set_sort_by_value (NOTMUCH_VALUE_TIMESTAMP, false);
            break;
        case NOTMUCH_SORT_NEWEST_FIRST:
-           enquire.set_sort_by_value (NOTMUCH_VALUE_TIMESTAMP, TRUE);
+           enquire.set_sort_by_value (NOTMUCH_VALUE_TIMESTAMP, true);
            break;
        case NOTMUCH_SORT_MESSAGE_ID:
-           enquire.set_sort_by_value (NOTMUCH_VALUE_MESSAGE_ID, FALSE);
+           enquire.set_sort_by_value (NOTMUCH_VALUE_MESSAGE_ID, false);
            break;
        case NOTMUCH_SORT_UNSORTED:
            break;
@@ -305,13 +359,13 @@ _notmuch_query_search_documents (notmuch_query_t *query,
                               "Query string was: %s\n",
                               query->query_string);
 
-       notmuch->exception_reported = TRUE;
+       notmuch->exception_reported = true;
        talloc_free (messages);
        return NOTMUCH_STATUS_XAPIAN_EXCEPTION;
     }
 }
 
-notmuch_bool_t
+bool
 _notmuch_mset_messages_valid (notmuch_messages_t *messages)
 {
     notmuch_mset_messages_t *mset_messages;
@@ -361,7 +415,7 @@ _notmuch_mset_messages_get (notmuch_messages_t *messages)
 
     if (messages->excluded_doc_ids &&
        _notmuch_doc_id_set_contains (messages->excluded_doc_ids, doc_id))
-       notmuch_message_set_flag (message, NOTMUCH_MESSAGE_FLAG_EXCLUDED, TRUE);
+       notmuch_message_set_flag (message, NOTMUCH_MESSAGE_FLAG_EXCLUDED, true);
 
     return message;
 }
@@ -376,7 +430,7 @@ _notmuch_mset_messages_move_to_next (notmuch_messages_t *messages)
     mset_messages->iterator++;
 }
 
-static notmuch_bool_t
+static bool
 _notmuch_doc_id_set_init (void *ctx,
                          notmuch_doc_id_set_t *doc_ids,
                          GArray *arr)
@@ -389,7 +443,7 @@ _notmuch_doc_id_set_init (void *ctx,
     bitmap = talloc_zero_array (ctx, unsigned char, DOCIDSET_WORD(max) + 1);
 
     if (bitmap == NULL)
-       return FALSE;
+       return false;
 
     doc_ids->bitmap = bitmap;
     doc_ids->bound = max + 1;
@@ -399,15 +453,15 @@ _notmuch_doc_id_set_init (void *ctx,
        bitmap[DOCIDSET_WORD(doc_id)] |= 1 << DOCIDSET_BIT(doc_id);
     }
 
-    return TRUE;
+    return true;
 }
 
-notmuch_bool_t
+bool
 _notmuch_doc_id_set_contains (notmuch_doc_id_set_t *doc_ids,
                              unsigned int doc_id)
 {
     if (doc_id >= doc_ids->bound)
-       return FALSE;
+       return false;
     return doc_ids->bitmap[DOCIDSET_WORD(doc_id)] & (1 << DOCIDSET_BIT(doc_id));
 }
 
@@ -432,22 +486,15 @@ _notmuch_threads_destructor (notmuch_threads_t *threads)
     return 0;
 }
 
-
-notmuch_threads_t *
-notmuch_query_search_threads (notmuch_query_t *query)
+notmuch_status_t
+notmuch_query_search_threads_st (notmuch_query_t *query, notmuch_threads_t **out)
 {
-    notmuch_status_t status;
-    notmuch_threads_t *threads;
-    status = notmuch_query_search_threads_st (query, &threads);
-    if (status)
-       return NULL;
-    else
-       return threads;
+    return notmuch_query_search_threads(query, out);
 }
 
 notmuch_status_t
-notmuch_query_search_threads_st (notmuch_query_t *query,
-                                notmuch_threads_t **out)
+notmuch_query_search_threads (notmuch_query_t *query,
+                             notmuch_threads_t **out)
 {
     notmuch_threads_t *threads;
     notmuch_messages_t *messages;
@@ -461,13 +508,13 @@ notmuch_query_search_threads_st (notmuch_query_t *query,
 
     threads->query = query;
 
-    status = notmuch_query_search_messages_st (query, &messages);
+    status = notmuch_query_search_messages (query, &messages);
     if (status) {
        talloc_free (threads);
        return status;
     }
 
-    threads->doc_ids = g_array_new (FALSE, FALSE, sizeof (unsigned int));
+    threads->doc_ids = g_array_new (false, false, sizeof (unsigned int));
     while (notmuch_messages_valid (messages)) {
        unsigned int doc_id = _notmuch_mset_messages_get_doc_id (messages);
        g_array_append_val (threads->doc_ids, doc_id);
@@ -499,7 +546,7 @@ notmuch_threads_valid (notmuch_threads_t *threads)
     unsigned int doc_id;
 
     if (! threads)
-       return FALSE;
+       return false;
 
     while (threads->doc_id_pos < threads->doc_ids->len) {
        doc_id = g_array_index (threads->doc_ids, unsigned int,
@@ -544,18 +591,14 @@ notmuch_threads_destroy (notmuch_threads_t *threads)
     talloc_free (threads);
 }
 
-unsigned int
-notmuch_query_count_messages (notmuch_query_t *query)
+notmuch_status_t
+notmuch_query_count_messages_st (notmuch_query_t *query, unsigned *count_out)
 {
-    notmuch_status_t status;
-    unsigned int count;
-
-    status = notmuch_query_count_messages_st (query, &count);
-    return status ? 0 : count;
+    return notmuch_query_count_messages (query, count_out);
 }
 
 notmuch_status_t
-notmuch_query_count_messages_st (notmuch_query_t *query, unsigned *count_out)
+notmuch_query_count_messages (notmuch_query_t *query, unsigned *count_out)
 {
     return _notmuch_query_count_documents (query, "mail", count_out);
 }
@@ -566,13 +609,18 @@ _notmuch_query_count_documents (notmuch_query_t *query, const char *type, unsign
     notmuch_database_t *notmuch = query->notmuch;
     const char *query_string = query->query_string;
     Xapian::doccount count = 0;
+    notmuch_status_t status;
+
+    status = _notmuch_query_ensure_parsed (query);
+    if (status)
+       return status;
 
     try {
        Xapian::Enquire enquire (*notmuch->xapian_db);
        Xapian::Query mail_query (talloc_asprintf (query, "%s%s",
                                                   _find_prefix ("type"),
                                                   type));
-       Xapian::Query string_query, final_query, exclude_query;
+       Xapian::Query final_query, exclude_query;
        Xapian::MSet mset;
 
        if (strcmp (query_string, "") == 0 ||
@@ -580,13 +628,11 @@ _notmuch_query_count_documents (notmuch_query_t *query, const char *type, unsign
        {
            final_query = mail_query;
        } else {
-           string_query = notmuch->query_parser->
-               parse_query (query_string, NOTMUCH_QUERY_PARSER_FLAGS);
            final_query = Xapian::Query (Xapian::Query::OP_AND,
-                                        mail_query, string_query);
+                                        mail_query, query->xapian_query);
        }
 
-       exclude_query = _notmuch_exclude_tags (query, final_query);
+       exclude_query = _notmuch_exclude_tags (query);
 
        final_query = Xapian::Query (Xapian::Query::OP_AND_NOT,
                                         final_query, exclude_query);
@@ -606,8 +652,9 @@ _notmuch_query_count_documents (notmuch_query_t *query, const char *type, unsign
        /*
         * Set the checkatleast parameter to the number of documents
         * in the database to make get_matches_estimated() exact.
+        * Set the max parameter to 1 to avoid fetching documents we will discard.
         */
-       mset = enquire.get_mset (0, notmuch->xapian_db->get_doccount (),
+       mset = enquire.get_mset (0, 1,
                                 notmuch->xapian_db->get_doccount ());
 
        count = mset.get_matches_estimated();
@@ -626,18 +673,14 @@ _notmuch_query_count_documents (notmuch_query_t *query, const char *type, unsign
     return NOTMUCH_STATUS_SUCCESS;
 }
 
-unsigned
-notmuch_query_count_threads (notmuch_query_t *query)
+notmuch_status_t
+notmuch_query_count_threads_st (notmuch_query_t *query, unsigned *count)
 {
-    notmuch_status_t status;
-    unsigned int count;
-
-    status = notmuch_query_count_threads_st (query, &count);
-    return status ? 0 : count;
+    return notmuch_query_count_threads (query, count);
 }
 
 notmuch_status_t
-notmuch_query_count_threads_st (notmuch_query_t *query, unsigned *count)
+notmuch_query_count_threads (notmuch_query_t *query, unsigned *count)
 {
     notmuch_messages_t *messages;
     GHashTable *hash;
@@ -646,7 +689,7 @@ notmuch_query_count_threads_st (notmuch_query_t *query, unsigned *count)
 
     sort = query->sort;
     query->sort = NOTMUCH_SORT_UNSORTED;
-    ret = notmuch_query_search_messages_st (query, &messages);
+    ret = notmuch_query_search_messages (query, &messages);
     if (ret)
        return ret;
     query->sort = sort;