X-Git-Url: https://git.notmuchmail.org/git?p=notmuch;a=blobdiff_plain;f=lib%2Fquery.cc;h=792aba219fa48d53e3b761089eac1d4349b5c0fb;hp=9c6ecc8db5ce0d4e6ecb1d4d5adfc1d65b715bc1;hb=HEAD;hpb=30c475c1ef3d6823ed9ab0affc30328726c5240f diff --git a/lib/query.cc b/lib/query.cc index 9c6ecc8d..1c60c122 100644 --- a/lib/query.cc +++ b/lib/query.cc @@ -20,6 +20,7 @@ #include "notmuch-private.h" #include "database-private.h" +#include "xapian-extra.h" #include /* GHashTable, GPtrArray */ @@ -29,7 +30,8 @@ struct _notmuch_query { notmuch_sort_t sort; notmuch_string_list_t *exclude_terms; notmuch_exclude_t omit_excluded; - notmuch_bool_t parsed; + bool parsed; + notmuch_query_syntax_t syntax; Xapian::Query xapian_query; std::set terms; }; @@ -62,29 +64,31 @@ struct _notmuch_threads { }; /* We need this in the message functions so forward declare. */ -static notmuch_bool_t +static bool _notmuch_doc_id_set_init (void *ctx, notmuch_doc_id_set_t *doc_ids, GArray *arr); -static notmuch_bool_t +static bool _debug_query (void) { char *env = getenv ("NOTMUCH_DEBUG_QUERY"); + return (env && strcmp (env, "") != 0); } /* Explicit destructor call for placement new */ static int -_notmuch_query_destructor (notmuch_query_t *query) { +_notmuch_query_destructor (notmuch_query_t *query) +{ query->xapian_query.~Query(); query->terms.~set(); return 0; } -notmuch_query_t * -notmuch_query_create (notmuch_database_t *notmuch, - const char *query_string) +static notmuch_query_t * +_notmuch_query_constructor (notmuch_database_t *notmuch, + const char *query_string) { notmuch_query_t *query; @@ -97,13 +101,16 @@ notmuch_query_create (notmuch_database_t *notmuch, new (&query->xapian_query) Xapian::Query (); new (&query->terms) std::set (); - query->parsed = FALSE; + query->parsed = false; talloc_set_destructor (query, _notmuch_query_destructor); query->notmuch = notmuch; - query->query_string = talloc_strdup (query, query_string); + if (query_string) + query->query_string = talloc_strdup (query, query_string); + else + query->query_string = NULL; query->sort = NOTMUCH_SORT_NEWEST_FIRST; @@ -114,44 +121,146 @@ notmuch_query_create (notmuch_database_t *notmuch, return query; } -static notmuch_status_t -_notmuch_query_ensure_parsed (notmuch_query_t *query) +notmuch_query_t * +notmuch_query_create (notmuch_database_t *notmuch, + const char *query_string) { - if (query->parsed) - return NOTMUCH_STATUS_SUCCESS; - try { - query->xapian_query = - query->notmuch->query_parser-> - parse_query (query->query_string, NOTMUCH_QUERY_PARSER_FLAGS); + notmuch_query_t *query; + notmuch_status_t status; - /* Xapian doesn't support skip_to on terms from a query since - * they are unordered, so cache a copy of all terms in - * something searchable. - */ + status = notmuch_query_create_with_syntax (notmuch, query_string, + NOTMUCH_QUERY_SYNTAX_XAPIAN, + &query); + if (status) + return NULL; + + return query; +} + +notmuch_status_t +notmuch_query_create_with_syntax (notmuch_database_t *notmuch, + const char *query_string, + notmuch_query_syntax_t syntax, + notmuch_query_t **output) +{ + + notmuch_query_t *query; + + if (! output) + return NOTMUCH_STATUS_NULL_POINTER; - for (Xapian::TermIterator t = query->xapian_query.get_terms_begin (); - t != query->xapian_query.get_terms_end (); ++t) - query->terms.insert (*t); + query = _notmuch_query_constructor (notmuch, query_string); + if (! query) + return NOTMUCH_STATUS_OUT_OF_MEMORY; + + if (syntax == NOTMUCH_QUERY_SYNTAX_SEXP && ! HAVE_SFSEXP) { + _notmuch_database_log (notmuch, "sexp query parser not available"); + return NOTMUCH_STATUS_ILLEGAL_ARGUMENT; + } - query->parsed = TRUE; + query->syntax = syntax; + *output = query; + + return NOTMUCH_STATUS_SUCCESS; +} + +static void +_notmuch_query_cache_terms (notmuch_query_t *query) +{ + /* Xapian doesn't support skip_to on terms from a query since + * they are unordered, so cache a copy of all terms in + * something searchable. + */ + + for (Xapian::TermIterator t = query->xapian_query.get_terms_begin (); + t != query->xapian_query.get_terms_end (); ++t) + query->terms.insert (*t); +} + +notmuch_status_t +_notmuch_query_string_to_xapian_query (notmuch_database_t *notmuch, + std::string query_string, + Xapian::Query &output, + std::string &msg) +{ + try { + if (query_string == "" || query_string == "*") { + output = xapian_query_match_all (); + } else { + output = + notmuch->query_parser-> + parse_query (query_string, NOTMUCH_QUERY_PARSER_FLAGS); + } } catch (const Xapian::Error &error) { - if (!query->notmuch->exception_reported) { - _notmuch_database_log (query->notmuch, + if (! notmuch->exception_reported) { + _notmuch_database_log (notmuch, "A Xapian exception occurred parsing query: %s\n", error.get_msg ().c_str ()); - _notmuch_database_log_append (query->notmuch, + _notmuch_database_log_append (notmuch, "Query string was: %s\n", - query->query_string); - query->notmuch->exception_reported = TRUE; + query_string.c_str ()); + notmuch->exception_reported = true; } + msg = error.get_msg (); return NOTMUCH_STATUS_XAPIAN_EXCEPTION; } return NOTMUCH_STATUS_SUCCESS; } +static notmuch_status_t +_notmuch_query_ensure_parsed_xapian (notmuch_query_t *query) +{ + notmuch_status_t status; + std::string msg; /* ignored */ + + status = _notmuch_query_string_to_xapian_query (query->notmuch, query->query_string, + query->xapian_query, msg); + if (status) + return status; + + query->parsed = true; + + _notmuch_query_cache_terms (query); + + return NOTMUCH_STATUS_SUCCESS; +} + +#if HAVE_SFSEXP +static notmuch_status_t +_notmuch_query_ensure_parsed_sexpr (notmuch_query_t *query) +{ + notmuch_status_t status; + + if (query->parsed) + return NOTMUCH_STATUS_SUCCESS; + + status = _notmuch_sexp_string_to_xapian_query (query->notmuch, query->query_string, + query->xapian_query); + if (status) + return status; + + _notmuch_query_cache_terms (query); + return NOTMUCH_STATUS_SUCCESS; +} +#endif + +static notmuch_status_t +_notmuch_query_ensure_parsed (notmuch_query_t *query) +{ + if (query->parsed) + return NOTMUCH_STATUS_SUCCESS; + +#if HAVE_SFSEXP + if (query->syntax == NOTMUCH_QUERY_SYNTAX_SEXP) + return _notmuch_query_ensure_parsed_sexpr (query); +#endif + + return _notmuch_query_ensure_parsed_xapian (query); +} + const char * notmuch_query_get_query_string (const notmuch_query_t *query) { @@ -188,7 +297,7 @@ notmuch_query_add_tag_exclude (notmuch_query_t *query, const char *tag) return status; term = talloc_asprintf (query, "%s%s", _find_prefix ("tag"), tag); - if (query->terms.count(term) != 0) + if (query->terms.count (term) != 0) return NOTMUCH_STATUS_IGNORED; _notmuch_string_list_append (query->exclude_terms, term); @@ -236,7 +345,7 @@ notmuch_query_search_messages_st (notmuch_query_t *query, notmuch_status_t notmuch_query_search_messages (notmuch_query_t *query, - notmuch_messages_t **out) + notmuch_messages_t **out) { return _notmuch_query_search_documents (query, "mail", out); } @@ -247,7 +356,6 @@ _notmuch_query_search_documents (notmuch_query_t *query, notmuch_messages_t **out) { notmuch_database_t *notmuch = query->notmuch; - const char *query_string = query->query_string; notmuch_mset_messages_t *messages; notmuch_status_t status; @@ -261,7 +369,7 @@ _notmuch_query_search_documents (notmuch_query_t *query, try { - messages->base.is_of_list_type = FALSE; + messages->base.is_of_list_type = false; messages->base.iterator = NULL; messages->notmuch = notmuch; new (&messages->iterator) Xapian::MSetIterator (); @@ -277,34 +385,28 @@ _notmuch_query_search_documents (notmuch_query_t *query, Xapian::MSet mset; Xapian::MSetIterator iterator; - if (strcmp (query_string, "") == 0 || - strcmp (query_string, "*") == 0) - { - final_query = mail_query; - } else { - final_query = Xapian::Query (Xapian::Query::OP_AND, - mail_query, query->xapian_query); - } + final_query = Xapian::Query (Xapian::Query::OP_AND, + mail_query, query->xapian_query); + messages->base.excluded_doc_ids = NULL; if ((query->omit_excluded != NOTMUCH_EXCLUDE_FALSE) && (query->exclude_terms)) { exclude_query = _notmuch_exclude_tags (query); if (query->omit_excluded == NOTMUCH_EXCLUDE_TRUE || - query->omit_excluded == NOTMUCH_EXCLUDE_ALL) - { + query->omit_excluded == NOTMUCH_EXCLUDE_ALL) { final_query = Xapian::Query (Xapian::Query::OP_AND_NOT, final_query, exclude_query); } else { /* NOTMUCH_EXCLUDE_FLAG */ exclude_query = Xapian::Query (Xapian::Query::OP_AND, - exclude_query, final_query); + exclude_query, final_query); - enquire.set_weighting_scheme (Xapian::BoolWeight()); + enquire.set_weighting_scheme (Xapian::BoolWeight ()); enquire.set_query (exclude_query); mset = enquire.get_mset (0, notmuch->xapian_db->get_doccount ()); - GArray *excluded_doc_ids = g_array_new (FALSE, FALSE, sizeof (unsigned int)); + GArray *excluded_doc_ids = g_array_new (false, false, sizeof (unsigned int)); for (iterator = mset.begin (); iterator != mset.end (); iterator++) { unsigned int doc_id = *iterator; @@ -318,17 +420,17 @@ _notmuch_query_search_documents (notmuch_query_t *query, } - enquire.set_weighting_scheme (Xapian::BoolWeight()); + enquire.set_weighting_scheme (Xapian::BoolWeight ()); switch (query->sort) { case NOTMUCH_SORT_OLDEST_FIRST: - enquire.set_sort_by_value (NOTMUCH_VALUE_TIMESTAMP, FALSE); + enquire.set_sort_by_value (NOTMUCH_VALUE_TIMESTAMP, false); break; case NOTMUCH_SORT_NEWEST_FIRST: - enquire.set_sort_by_value (NOTMUCH_VALUE_TIMESTAMP, TRUE); + enquire.set_sort_by_value (NOTMUCH_VALUE_TIMESTAMP, true); break; case NOTMUCH_SORT_MESSAGE_ID: - enquire.set_sort_by_value (NOTMUCH_VALUE_MESSAGE_ID, FALSE); + enquire.set_sort_by_value (NOTMUCH_VALUE_MESSAGE_ID, false); break; case NOTMUCH_SORT_UNSORTED: break; @@ -354,18 +456,18 @@ _notmuch_query_search_documents (notmuch_query_t *query, } catch (const Xapian::Error &error) { _notmuch_database_log (notmuch, "A Xapian exception occurred performing query: %s\n", - error.get_msg().c_str()); + error.get_msg ().c_str ()); _notmuch_database_log_append (notmuch, - "Query string was: %s\n", - query->query_string); + "Query string was: %s\n", + query->query_string); - notmuch->exception_reported = TRUE; + notmuch->exception_reported = true; talloc_free (messages); return NOTMUCH_STATUS_XAPIAN_EXCEPTION; } } -notmuch_bool_t +bool _notmuch_mset_messages_valid (notmuch_messages_t *messages) { notmuch_mset_messages_t *mset_messages; @@ -408,14 +510,13 @@ _notmuch_mset_messages_get (notmuch_messages_t *messages) &status); if (message == NULL && - status == NOTMUCH_PRIVATE_STATUS_NO_DOCUMENT_FOUND) - { + status == NOTMUCH_PRIVATE_STATUS_NO_DOCUMENT_FOUND) { INTERNAL_ERROR ("a messages iterator contains a non-existent document ID.\n"); } if (messages->excluded_doc_ids && _notmuch_doc_id_set_contains (messages->excluded_doc_ids, doc_id)) - notmuch_message_set_flag (message, NOTMUCH_MESSAGE_FLAG_EXCLUDED, TRUE); + notmuch_message_set_flag (message, NOTMUCH_MESSAGE_FLAG_EXCLUDED, true); return message; } @@ -430,7 +531,7 @@ _notmuch_mset_messages_move_to_next (notmuch_messages_t *messages) mset_messages->iterator++; } -static notmuch_bool_t +static bool _notmuch_doc_id_set_init (void *ctx, notmuch_doc_id_set_t *doc_ids, GArray *arr) @@ -439,30 +540,30 @@ _notmuch_doc_id_set_init (void *ctx, unsigned char *bitmap; for (unsigned int i = 0; i < arr->len; i++) - max = MAX(max, g_array_index (arr, unsigned int, i)); - bitmap = talloc_zero_array (ctx, unsigned char, DOCIDSET_WORD(max) + 1); + max = MAX (max, g_array_index (arr, unsigned int, i)); + bitmap = talloc_zero_array (ctx, unsigned char, DOCIDSET_WORD (max) + 1); if (bitmap == NULL) - return FALSE; + return false; doc_ids->bitmap = bitmap; doc_ids->bound = max + 1; for (unsigned int i = 0; i < arr->len; i++) { unsigned int doc_id = g_array_index (arr, unsigned int, i); - bitmap[DOCIDSET_WORD(doc_id)] |= 1 << DOCIDSET_BIT(doc_id); + bitmap[DOCIDSET_WORD (doc_id)] |= 1 << DOCIDSET_BIT (doc_id); } - return TRUE; + return true; } -notmuch_bool_t +bool _notmuch_doc_id_set_contains (notmuch_doc_id_set_t *doc_ids, unsigned int doc_id) { if (doc_id >= doc_ids->bound) - return FALSE; - return doc_ids->bitmap[DOCIDSET_WORD(doc_id)] & (1 << DOCIDSET_BIT(doc_id)); + return false; + return doc_ids->bitmap[DOCIDSET_WORD (doc_id)] & (1 << DOCIDSET_BIT (doc_id)); } void @@ -470,7 +571,7 @@ _notmuch_doc_id_set_remove (notmuch_doc_id_set_t *doc_ids, unsigned int doc_id) { if (doc_id < doc_ids->bound) - doc_ids->bitmap[DOCIDSET_WORD(doc_id)] &= ~(1 << DOCIDSET_BIT(doc_id)); + doc_ids->bitmap[DOCIDSET_WORD (doc_id)] &= ~(1 << DOCIDSET_BIT (doc_id)); } /* Glib objects force use to use a talloc destructor as well, (but not @@ -489,7 +590,7 @@ _notmuch_threads_destructor (notmuch_threads_t *threads) notmuch_status_t notmuch_query_search_threads_st (notmuch_query_t *query, notmuch_threads_t **out) { - return notmuch_query_search_threads(query, out); + return notmuch_query_search_threads (query, out); } notmuch_status_t @@ -514,7 +615,7 @@ notmuch_query_search_threads (notmuch_query_t *query, return status; } - threads->doc_ids = g_array_new (FALSE, FALSE, sizeof (unsigned int)); + threads->doc_ids = g_array_new (false, false, sizeof (unsigned int)); while (notmuch_messages_valid (messages)) { unsigned int doc_id = _notmuch_mset_messages_get_doc_id (messages); g_array_append_val (threads->doc_ids, doc_id); @@ -546,7 +647,7 @@ notmuch_threads_valid (notmuch_threads_t *threads) unsigned int doc_id; if (! threads) - return FALSE; + return false; while (threads->doc_id_pos < threads->doc_ids->len) { doc_id = g_array_index (threads->doc_ids, unsigned int, @@ -607,7 +708,6 @@ notmuch_status_t _notmuch_query_count_documents (notmuch_query_t *query, const char *type, unsigned *count_out) { notmuch_database_t *notmuch = query->notmuch; - const char *query_string = query->query_string; Xapian::doccount count = 0; notmuch_status_t status; @@ -623,22 +723,16 @@ _notmuch_query_count_documents (notmuch_query_t *query, const char *type, unsign Xapian::Query final_query, exclude_query; Xapian::MSet mset; - if (strcmp (query_string, "") == 0 || - strcmp (query_string, "*") == 0) - { - final_query = mail_query; - } else { - final_query = Xapian::Query (Xapian::Query::OP_AND, - mail_query, query->xapian_query); - } + final_query = Xapian::Query (Xapian::Query::OP_AND, + mail_query, query->xapian_query); exclude_query = _notmuch_exclude_tags (query); final_query = Xapian::Query (Xapian::Query::OP_AND_NOT, - final_query, exclude_query); + final_query, exclude_query); - enquire.set_weighting_scheme(Xapian::BoolWeight()); - enquire.set_docid_order(Xapian::Enquire::ASCENDING); + enquire.set_weighting_scheme (Xapian::BoolWeight ()); + enquire.set_docid_order (Xapian::Enquire::ASCENDING); if (_debug_query ()) { fprintf (stderr, "Exclude query is:\n%s\n", @@ -652,17 +746,17 @@ _notmuch_query_count_documents (notmuch_query_t *query, const char *type, unsign /* * Set the checkatleast parameter to the number of documents * in the database to make get_matches_estimated() exact. - * Set the max parameter to 0 to avoid fetching documents we will discard. + * Set the max parameter to 1 to avoid fetching documents we will discard. */ - mset = enquire.get_mset (0, 0, + mset = enquire.get_mset (0, 1, notmuch->xapian_db->get_doccount ()); - count = mset.get_matches_estimated(); + count = mset.get_matches_estimated (); } catch (const Xapian::Error &error) { _notmuch_database_log (notmuch, "A Xapian exception occurred performing query: %s\n", - error.get_msg().c_str()); + error.get_msg ().c_str ()); _notmuch_database_log_append (notmuch, "Query string was: %s\n", query->query_string); @@ -730,3 +824,51 @@ notmuch_query_get_database (const notmuch_query_t *query) { return query->notmuch; } + +notmuch_status_t +_notmuch_query_expand (notmuch_database_t *notmuch, const char *field, Xapian::Query subquery, + Xapian::Query &output, std::string &msg) +{ + std::set terms; + const std::string term_prefix = _find_prefix (field); + + if (_debug_query ()) { + fprintf (stderr, "Expanding subquery:\n%s\n", + subquery.get_description ().c_str ()); + } + + try { + Xapian::Enquire enquire (*notmuch->xapian_db); + Xapian::MSet mset; + + enquire.set_weighting_scheme (Xapian::BoolWeight ()); + enquire.set_query (subquery); + + mset = enquire.get_mset (0, notmuch->xapian_db->get_doccount ()); + + for (Xapian::MSetIterator iterator = mset.begin (); iterator != mset.end (); iterator++) { + Xapian::docid doc_id = *iterator; + Xapian::Document doc = notmuch->xapian_db->get_document (doc_id); + Xapian::TermIterator i = doc.termlist_begin (); + + for (i.skip_to (term_prefix); + i != doc.termlist_end () && ((*i).rfind (term_prefix, 0) == 0); i++) { + terms.insert (*i); + } + } + output = Xapian::Query (Xapian::Query::OP_OR, terms.begin (), terms.end ()); + if (_debug_query ()) { + fprintf (stderr, "Expanded query:\n%s\n", + subquery.get_description ().c_str ()); + } + + } catch (const Xapian::Error &error) { + _notmuch_database_log (notmuch, + "A Xapian exception occurred expanding query: %s\n", + error.get_msg ().c_str ()); + msg = error.get_msg (); + return NOTMUCH_STATUS_XAPIAN_EXCEPTION; + } + + return NOTMUCH_STATUS_SUCCESS; +}