git.notmuchmail.org Git - notmuch/blob - lib/query.cc

   1 /* query.cc - Support for searching a notmuch database
   2  *
   3  * Copyright © 2009 Carl Worth
   4  *
   5  * This program is free software: you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License as published by
   7  * the Free Software Foundation, either version 3 of the License, or
   8  * (at your option) any later version.
   9  *
  10  * This program is distributed in the hope that it will be useful,
  11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13  * GNU General Public License for more details.
  14  *
  15  * You should have received a copy of the GNU General Public License
  16  * along with this program.  If not, see https://www.gnu.org/licenses/ .
  17  *
  18  * Author: Carl Worth <cworth@cworth.org>
  19  */
  20
  21 #include "notmuch-private.h"
  22 #include "database-private.h"
  23 #include "xapian-extra.h"
  24
  25 #include <glib.h> /* GHashTable, GPtrArray */
  26
  27 struct _notmuch_query {
  28     notmuch_database_t *notmuch;
  29     const char *query_string;
  30     notmuch_sort_t sort;
  31     notmuch_string_list_t *exclude_terms;
  32     notmuch_exclude_t omit_excluded;
  33     bool parsed;
  34     notmuch_query_syntax_t syntax;
  35     Xapian::Query xapian_query;
  36     std::set<std::string> terms;
  37 };
  38
  39 typedef struct _notmuch_mset_messages {
  40     notmuch_messages_t base;
  41     notmuch_database_t *notmuch;
  42     Xapian::MSetIterator iterator;
  43     Xapian::MSetIterator iterator_end;
  44 } notmuch_mset_messages_t;
  45
  46 struct _notmuch_doc_id_set {
  47     unsigned char *bitmap;
  48     unsigned int bound;
  49 };
  50
  51 #define DOCIDSET_WORD(bit) ((bit) / CHAR_BIT)
  52 #define DOCIDSET_BIT(bit) ((bit) % CHAR_BIT)
  53
  54 struct _notmuch_threads {
  55     notmuch_query_t *query;
  56
  57     /* The ordered list of doc ids matched by the query. */
  58     GArray *doc_ids;
  59     /* Our iterator's current position in doc_ids. */
  60     unsigned int doc_id_pos;
  61     /* The set of matched docid's that have not been assigned to a
  62      * thread. Initially, this contains every docid in doc_ids. */
  63     notmuch_doc_id_set_t match_set;
  64 };
  65
  66 /* We need this in the message functions so forward declare. */
  67 static bool
  68 _notmuch_doc_id_set_init (void *ctx,
  69                           notmuch_doc_id_set_t *doc_ids,
  70                           GArray *arr);
  71
  72 static bool
  73 _debug_query (void)
  74 {
  75     char *env = getenv ("NOTMUCH_DEBUG_QUERY");
  76
  77     return (env && strcmp (env, "") != 0);
  78 }
  79
  80 /* Explicit destructor call for placement new */
  81 static int
  82 _notmuch_query_destructor (notmuch_query_t *query)
  83 {
  84     query->xapian_query.~Query();
  85     query->terms.~set<std::string>();
  86     return 0;
  87 }
  88
  89 static notmuch_query_t *
  90 _notmuch_query_constructor (notmuch_database_t *notmuch,
  91                             const char *query_string)
  92 {
  93     notmuch_query_t *query;
  94
  95     if (_debug_query ())
  96         fprintf (stderr, "Query string is:\n%s\n", query_string);
  97
  98     query = talloc (notmuch, notmuch_query_t);
  99     if (unlikely (query == NULL))
 100         return NULL;
 101
 102     new (&query->xapian_query) Xapian::Query ();
 103     new (&query->terms) std::set<std::string> ();
 104     query->parsed = false;
 105
 106     talloc_set_destructor (query, _notmuch_query_destructor);
 107
 108     query->notmuch = notmuch;
 109
 110     if (query_string)
 111         query->query_string = talloc_strdup (query, query_string);
 112     else
 113         query->query_string = NULL;
 114
 115     query->sort = NOTMUCH_SORT_NEWEST_FIRST;
 116
 117     query->exclude_terms = _notmuch_string_list_create (query);
 118
 119     query->omit_excluded = NOTMUCH_EXCLUDE_TRUE;
 120
 121     return query;
 122 }
 123
 124 notmuch_query_t *
 125 notmuch_query_create (notmuch_database_t *notmuch,
 126                       const char *query_string)
 127 {
 128
 129     notmuch_query_t *query;
 130     notmuch_status_t status;
 131
 132     status = notmuch_query_create_with_syntax (notmuch, query_string,
 133                                                NOTMUCH_QUERY_SYNTAX_XAPIAN,
 134                                                &query);
 135     if (status)
 136         return NULL;
 137
 138     return query;
 139 }
 140
 141 notmuch_status_t
 142 notmuch_query_create_with_syntax (notmuch_database_t *notmuch,
 143                                   const char *query_string,
 144                                   notmuch_query_syntax_t syntax,
 145                                   notmuch_query_t **output)
 146 {
 147
 148     notmuch_query_t *query;
 149
 150     if (! output)
 151         return NOTMUCH_STATUS_NULL_POINTER;
 152
 153     query = _notmuch_query_constructor (notmuch, query_string);
 154     if (! query)
 155         return NOTMUCH_STATUS_OUT_OF_MEMORY;
 156
 157     if (syntax == NOTMUCH_QUERY_SYNTAX_SEXP && ! HAVE_SFSEXP) {
 158         _notmuch_database_log (notmuch, "sexp query parser not available");
 159         return NOTMUCH_STATUS_ILLEGAL_ARGUMENT;
 160     }
 161
 162     query->syntax = syntax;
 163
 164     *output = query;
 165
 166     return NOTMUCH_STATUS_SUCCESS;
 167 }
 168
 169 static void
 170 _notmuch_query_cache_terms (notmuch_query_t *query)
 171 {
 172     /* Xapian doesn't support skip_to on terms from a query since
 173      *  they are unordered, so cache a copy of all terms in
 174      *  something searchable.
 175      */
 176
 177     for (Xapian::TermIterator t = query->xapian_query.get_terms_begin ();
 178          t != query->xapian_query.get_terms_end (); ++t)
 179         query->terms.insert (*t);
 180 }
 181
 182 notmuch_status_t
 183 _notmuch_query_string_to_xapian_query (notmuch_database_t *notmuch,
 184                                        std::string query_string,
 185                                        Xapian::Query &output,
 186                                        std::string &msg)
 187 {
 188     try {
 189         if (query_string == "" || query_string == "*") {
 190             output = xapian_query_match_all ();
 191         } else {
 192             output =
 193                 notmuch->query_parser->
 194                 parse_query (query_string, NOTMUCH_QUERY_PARSER_FLAGS);
 195         }
 196     } catch (const Xapian::Error &error) {
 197         if (! notmuch->exception_reported) {
 198             _notmuch_database_log (notmuch,
 199                                    "A Xapian exception occurred parsing query: %s\n",
 200                                    error.get_msg ().c_str ());
 201             _notmuch_database_log_append (notmuch,
 202                                           "Query string was: %s\n",
 203                                           query_string.c_str ());
 204             notmuch->exception_reported = true;
 205         }
 206
 207         msg = error.get_msg ();
 208         return NOTMUCH_STATUS_XAPIAN_EXCEPTION;
 209     }
 210     return NOTMUCH_STATUS_SUCCESS;
 211 }
 212
 213 static notmuch_status_t
 214 _notmuch_query_ensure_parsed_xapian (notmuch_query_t *query)
 215 {
 216     notmuch_status_t status;
 217     std::string msg; /* ignored */
 218
 219     status =  _notmuch_query_string_to_xapian_query (query->notmuch, query->query_string,
 220                                                      query->xapian_query, msg);
 221     if (status)
 222         return status;
 223
 224     query->parsed = true;
 225
 226     _notmuch_query_cache_terms (query);
 227
 228     return NOTMUCH_STATUS_SUCCESS;
 229 }
 230
 231 #if HAVE_SFSEXP
 232 static notmuch_status_t
 233 _notmuch_query_ensure_parsed_sexpr (notmuch_query_t *query)
 234 {
 235     notmuch_status_t status;
 236
 237     if (query->parsed)
 238         return NOTMUCH_STATUS_SUCCESS;
 239
 240     status = _notmuch_sexp_string_to_xapian_query (query->notmuch, query->query_string,
 241                                                    query->xapian_query);
 242     if (status)
 243         return status;
 244
 245     _notmuch_query_cache_terms (query);
 246     return NOTMUCH_STATUS_SUCCESS;
 247 }
 248 #endif
 249
 250 static notmuch_status_t
 251 _notmuch_query_ensure_parsed (notmuch_query_t *query)
 252 {
 253     if (query->parsed)
 254         return NOTMUCH_STATUS_SUCCESS;
 255
 256 #if HAVE_SFSEXP
 257     if (query->syntax == NOTMUCH_QUERY_SYNTAX_SEXP)
 258         return _notmuch_query_ensure_parsed_sexpr (query);
 259 #endif
 260
 261     return _notmuch_query_ensure_parsed_xapian (query);
 262 }
 263
 264 const char *
 265 notmuch_query_get_query_string (const notmuch_query_t *query)
 266 {
 267     return query->query_string;
 268 }
 269
 270 void
 271 notmuch_query_set_omit_excluded (notmuch_query_t *query,
 272                                  notmuch_exclude_t omit_excluded)
 273 {
 274     query->omit_excluded = omit_excluded;
 275 }
 276
 277 void
 278 notmuch_query_set_sort (notmuch_query_t *query, notmuch_sort_t sort)
 279 {
 280     query->sort = sort;
 281 }
 282
 283 notmuch_sort_t
 284 notmuch_query_get_sort (const notmuch_query_t *query)
 285 {
 286     return query->sort;
 287 }
 288
 289 notmuch_status_t
 290 notmuch_query_add_tag_exclude (notmuch_query_t *query, const char *tag)
 291 {
 292     notmuch_status_t status;
 293     char *term;
 294
 295     status = _notmuch_query_ensure_parsed (query);
 296     if (status)
 297         return status;
 298
 299     term = talloc_asprintf (query, "%s%s", _find_prefix ("tag"), tag);
 300     if (query->terms.count (term) != 0)
 301         return NOTMUCH_STATUS_IGNORED;
 302
 303     _notmuch_string_list_append (query->exclude_terms, term);
 304     return NOTMUCH_STATUS_SUCCESS;
 305 }
 306
 307 /* We end up having to call the destructors explicitly because we had
 308  * to use "placement new" in order to initialize C++ objects within a
 309  * block that we allocated with talloc. So C++ is making talloc
 310  * slightly less simple to use, (we wouldn't need
 311  * talloc_set_destructor at all otherwise).
 312  */
 313 static int
 314 _notmuch_messages_destructor (notmuch_mset_messages_t *messages)
 315 {
 316     messages->iterator.~MSetIterator ();
 317     messages->iterator_end.~MSetIterator ();
 318
 319     return 0;
 320 }
 321
 322 /* Return a query that matches messages with the excluded tags
 323  * registered with query. The caller of this function has to combine the returned
 324  * query appropriately.*/
 325 static Xapian::Query
 326 _notmuch_exclude_tags (notmuch_query_t *query)
 327 {
 328     Xapian::Query exclude_query = Xapian::Query::MatchNothing;
 329
 330     for (notmuch_string_node_t *term = query->exclude_terms->head; term;
 331          term = term->next) {
 332         exclude_query = Xapian::Query (Xapian::Query::OP_OR,
 333                                        exclude_query, Xapian::Query (term->string));
 334     }
 335     return exclude_query;
 336 }
 337
 338
 339 notmuch_status_t
 340 notmuch_query_search_messages_st (notmuch_query_t *query,
 341                                   notmuch_messages_t **out)
 342 {
 343     return notmuch_query_search_messages (query, out);
 344 }
 345
 346 notmuch_status_t
 347 notmuch_query_search_messages (notmuch_query_t *query,
 348                                notmuch_messages_t **out)
 349 {
 350     return _notmuch_query_search_documents (query, "mail", out);
 351 }
 352
 353 notmuch_status_t
 354 _notmuch_query_search_documents (notmuch_query_t *query,
 355                                  const char *type,
 356                                  notmuch_messages_t **out)
 357 {
 358     notmuch_database_t *notmuch = query->notmuch;
 359     notmuch_mset_messages_t *messages;
 360     notmuch_status_t status;
 361
 362     status = _notmuch_query_ensure_parsed (query);
 363     if (status)
 364         return status;
 365
 366     messages = talloc (query, notmuch_mset_messages_t);
 367     if (unlikely (messages == NULL))
 368         return NOTMUCH_STATUS_OUT_OF_MEMORY;
 369
 370     try {
 371
 372         messages->base.is_of_list_type = false;
 373         messages->base.iterator = NULL;
 374         messages->notmuch = notmuch;
 375         new (&messages->iterator) Xapian::MSetIterator ();
 376         new (&messages->iterator_end) Xapian::MSetIterator ();
 377
 378         talloc_set_destructor (messages, _notmuch_messages_destructor);
 379
 380         Xapian::Enquire enquire (*notmuch->xapian_db);
 381         Xapian::Query mail_query (talloc_asprintf (query, "%s%s",
 382                                                    _find_prefix ("type"),
 383                                                    type));
 384         Xapian::Query final_query, exclude_query;
 385         Xapian::MSet mset;
 386         Xapian::MSetIterator iterator;
 387
 388         final_query = Xapian::Query (Xapian::Query::OP_AND,
 389                                      mail_query, query->xapian_query);
 390
 391         messages->base.excluded_doc_ids = NULL;
 392
 393         if ((query->omit_excluded != NOTMUCH_EXCLUDE_FALSE) && (query->exclude_terms)) {
 394             exclude_query = _notmuch_exclude_tags (query);
 395
 396             if (query->omit_excluded == NOTMUCH_EXCLUDE_TRUE ||
 397                 query->omit_excluded == NOTMUCH_EXCLUDE_ALL) {
 398                 final_query = Xapian::Query (Xapian::Query::OP_AND_NOT,
 399                                              final_query, exclude_query);
 400             } else { /* NOTMUCH_EXCLUDE_FLAG */
 401                 exclude_query = Xapian::Query (Xapian::Query::OP_AND,
 402                                                exclude_query, final_query);
 403
 404                 enquire.set_weighting_scheme (Xapian::BoolWeight ());
 405                 enquire.set_query (exclude_query);
 406
 407                 mset = enquire.get_mset (0, notmuch->xapian_db->get_doccount ());
 408
 409                 GArray *excluded_doc_ids = g_array_new (false, false, sizeof (unsigned int));
 410
 411                 for (iterator = mset.begin (); iterator != mset.end (); iterator++) {
 412                     unsigned int doc_id = *iterator;
 413                     g_array_append_val (excluded_doc_ids, doc_id);
 414                 }
 415                 messages->base.excluded_doc_ids = talloc (messages, _notmuch_doc_id_set);
 416                 _notmuch_doc_id_set_init (query, messages->base.excluded_doc_ids,
 417                                           excluded_doc_ids);
 418                 g_array_unref (excluded_doc_ids);
 419             }
 420         }
 421
 422
 423         enquire.set_weighting_scheme (Xapian::BoolWeight ());
 424
 425         switch (query->sort) {
 426         case NOTMUCH_SORT_OLDEST_FIRST:
 427             enquire.set_sort_by_value (NOTMUCH_VALUE_TIMESTAMP, false);
 428             break;
 429         case NOTMUCH_SORT_NEWEST_FIRST:
 430             enquire.set_sort_by_value (NOTMUCH_VALUE_TIMESTAMP, true);
 431             break;
 432         case NOTMUCH_SORT_MESSAGE_ID:
 433             enquire.set_sort_by_value (NOTMUCH_VALUE_MESSAGE_ID, false);
 434             break;
 435         case NOTMUCH_SORT_UNSORTED:
 436             break;
 437         }
 438
 439         if (_debug_query ()) {
 440             fprintf (stderr, "Exclude query is:\n%s\n",
 441                      exclude_query.get_description ().c_str ());
 442             fprintf (stderr, "Final query is:\n%s\n",
 443                      final_query.get_description ().c_str ());
 444         }
 445
 446         enquire.set_query (final_query);
 447
 448         mset = enquire.get_mset (0, notmuch->xapian_db->get_doccount ());
 449
 450         messages->iterator = mset.begin ();
 451         messages->iterator_end = mset.end ();
 452
 453         *out = &messages->base;
 454         return NOTMUCH_STATUS_SUCCESS;
 455
 456     } catch (const Xapian::Error &error) {
 457         _notmuch_database_log (notmuch,
 458                                "A Xapian exception occurred performing query: %s\n",
 459                                error.get_msg ().c_str ());
 460         _notmuch_database_log_append (notmuch,
 461                                       "Query string was: %s\n",
 462                                       query->query_string);
 463
 464         notmuch->exception_reported = true;
 465         talloc_free (messages);
 466         return NOTMUCH_STATUS_XAPIAN_EXCEPTION;
 467     }
 468 }
 469
 470 bool
 471 _notmuch_mset_messages_valid (notmuch_messages_t *messages)
 472 {
 473     notmuch_mset_messages_t *mset_messages;
 474
 475     mset_messages = (notmuch_mset_messages_t *) messages;
 476
 477     return (mset_messages->iterator != mset_messages->iterator_end);
 478 }
 479
 480 static Xapian::docid
 481 _notmuch_mset_messages_get_doc_id (notmuch_messages_t *messages)
 482 {
 483     notmuch_mset_messages_t *mset_messages;
 484
 485     mset_messages = (notmuch_mset_messages_t *) messages;
 486
 487     if (! _notmuch_mset_messages_valid (&mset_messages->base))
 488         return 0;
 489
 490     return *mset_messages->iterator;
 491 }
 492
 493 notmuch_message_t *
 494 _notmuch_mset_messages_get (notmuch_messages_t *messages)
 495 {
 496     notmuch_message_t *message;
 497     Xapian::docid doc_id;
 498     notmuch_private_status_t status;
 499     notmuch_mset_messages_t *mset_messages;
 500
 501     mset_messages = (notmuch_mset_messages_t *) messages;
 502
 503     if (! _notmuch_mset_messages_valid (&mset_messages->base))
 504         return NULL;
 505
 506     doc_id = *mset_messages->iterator;
 507
 508     message = _notmuch_message_create (mset_messages,
 509                                        mset_messages->notmuch, doc_id,
 510                                        &status);
 511
 512     if (message == NULL &&
 513         status == NOTMUCH_PRIVATE_STATUS_NO_DOCUMENT_FOUND) {
 514         INTERNAL_ERROR ("a messages iterator contains a non-existent document ID.\n");
 515     }
 516
 517     if (messages->excluded_doc_ids &&
 518         _notmuch_doc_id_set_contains (messages->excluded_doc_ids, doc_id))
 519         notmuch_message_set_flag (message, NOTMUCH_MESSAGE_FLAG_EXCLUDED, true);
 520
 521     return message;
 522 }
 523
 524 void
 525 _notmuch_mset_messages_move_to_next (notmuch_messages_t *messages)
 526 {
 527     notmuch_mset_messages_t *mset_messages;
 528
 529     mset_messages = (notmuch_mset_messages_t *) messages;
 530
 531     mset_messages->iterator++;
 532 }
 533
 534 static bool
 535 _notmuch_doc_id_set_init (void *ctx,
 536                           notmuch_doc_id_set_t *doc_ids,
 537                           GArray *arr)
 538 {
 539     unsigned int max = 0;
 540     unsigned char *bitmap;
 541
 542     for (unsigned int i = 0; i < arr->len; i++)
 543         max = MAX (max, g_array_index (arr, unsigned int, i));
 544     bitmap = talloc_zero_array (ctx, unsigned char, DOCIDSET_WORD (max) + 1);
 545
 546     if (bitmap == NULL)
 547         return false;
 548
 549     doc_ids->bitmap = bitmap;
 550     doc_ids->bound = max + 1;
 551
 552     for (unsigned int i = 0; i < arr->len; i++) {
 553         unsigned int doc_id = g_array_index (arr, unsigned int, i);
 554         bitmap[DOCIDSET_WORD (doc_id)] |= 1 << DOCIDSET_BIT (doc_id);
 555     }
 556
 557     return true;
 558 }
 559
 560 bool
 561 _notmuch_doc_id_set_contains (notmuch_doc_id_set_t *doc_ids,
 562                               unsigned int doc_id)
 563 {
 564     if (doc_id >= doc_ids->bound)
 565         return false;
 566     return doc_ids->bitmap[DOCIDSET_WORD (doc_id)] & (1 << DOCIDSET_BIT (doc_id));
 567 }
 568
 569 void
 570 _notmuch_doc_id_set_remove (notmuch_doc_id_set_t *doc_ids,
 571                             unsigned int doc_id)
 572 {
 573     if (doc_id < doc_ids->bound)
 574         doc_ids->bitmap[DOCIDSET_WORD (doc_id)] &= ~(1 << DOCIDSET_BIT (doc_id));
 575 }
 576
 577 /* Glib objects force use to use a talloc destructor as well, (but not
 578  * nearly as ugly as the for messages due to C++ objects). At
 579  * this point, I'd really like to have some talloc-friendly
 580  * equivalents for the few pieces of glib that I'm using. */
 581 static int
 582 _notmuch_threads_destructor (notmuch_threads_t *threads)
 583 {
 584     if (threads->doc_ids)
 585         g_array_unref (threads->doc_ids);
 586
 587     return 0;
 588 }
 589
 590 notmuch_status_t
 591 notmuch_query_search_threads_st (notmuch_query_t *query, notmuch_threads_t **out)
 592 {
 593     return notmuch_query_search_threads (query, out);
 594 }
 595
 596 notmuch_status_t
 597 notmuch_query_search_threads (notmuch_query_t *query,
 598                               notmuch_threads_t **out)
 599 {
 600     notmuch_threads_t *threads;
 601     notmuch_messages_t *messages;
 602     notmuch_status_t status;
 603
 604     threads = talloc (query, notmuch_threads_t);
 605     if (threads == NULL)
 606         return NOTMUCH_STATUS_OUT_OF_MEMORY;
 607     threads->doc_ids = NULL;
 608     talloc_set_destructor (threads, _notmuch_threads_destructor);
 609
 610     threads->query = query;
 611
 612     status = notmuch_query_search_messages (query, &messages);
 613     if (status) {
 614         talloc_free (threads);
 615         return status;
 616     }
 617
 618     threads->doc_ids = g_array_new (false, false, sizeof (unsigned int));
 619     while (notmuch_messages_valid (messages)) {
 620         unsigned int doc_id = _notmuch_mset_messages_get_doc_id (messages);
 621         g_array_append_val (threads->doc_ids, doc_id);
 622         notmuch_messages_move_to_next (messages);
 623     }
 624     threads->doc_id_pos = 0;
 625
 626     talloc_free (messages);
 627
 628     if (! _notmuch_doc_id_set_init (threads, &threads->match_set,
 629                                     threads->doc_ids)) {
 630         talloc_free (threads);
 631         return NOTMUCH_STATUS_OUT_OF_MEMORY;
 632     }
 633
 634     *out = threads;
 635     return NOTMUCH_STATUS_SUCCESS;
 636 }
 637
 638 void
 639 notmuch_query_destroy (notmuch_query_t *query)
 640 {
 641     talloc_free (query);
 642 }
 643
 644 notmuch_bool_t
 645 notmuch_threads_valid (notmuch_threads_t *threads)
 646 {
 647     unsigned int doc_id;
 648
 649     if (! threads)
 650         return false;
 651
 652     while (threads->doc_id_pos < threads->doc_ids->len) {
 653         doc_id = g_array_index (threads->doc_ids, unsigned int,
 654                                 threads->doc_id_pos);
 655         if (_notmuch_doc_id_set_contains (&threads->match_set, doc_id))
 656             break;
 657
 658         threads->doc_id_pos++;
 659     }
 660
 661     return threads->doc_id_pos < threads->doc_ids->len;
 662 }
 663
 664 notmuch_thread_t *
 665 notmuch_threads_get (notmuch_threads_t *threads)
 666 {
 667     unsigned int doc_id;
 668
 669     if (! notmuch_threads_valid (threads))
 670         return NULL;
 671
 672     doc_id = g_array_index (threads->doc_ids, unsigned int,
 673                             threads->doc_id_pos);
 674     return _notmuch_thread_create (threads->query,
 675                                    threads->query->notmuch,
 676                                    doc_id,
 677                                    &threads->match_set,
 678                                    threads->query->exclude_terms,
 679                                    threads->query->omit_excluded,
 680                                    threads->query->sort);
 681 }
 682
 683 void
 684 notmuch_threads_move_to_next (notmuch_threads_t *threads)
 685 {
 686     threads->doc_id_pos++;
 687 }
 688
 689 void
 690 notmuch_threads_destroy (notmuch_threads_t *threads)
 691 {
 692     talloc_free (threads);
 693 }
 694
 695 notmuch_status_t
 696 notmuch_query_count_messages_st (notmuch_query_t *query, unsigned *count_out)
 697 {
 698     return notmuch_query_count_messages (query, count_out);
 699 }
 700
 701 notmuch_status_t
 702 notmuch_query_count_messages (notmuch_query_t *query, unsigned *count_out)
 703 {
 704     return _notmuch_query_count_documents (query, "mail", count_out);
 705 }
 706
 707 notmuch_status_t
 708 _notmuch_query_count_documents (notmuch_query_t *query, const char *type, unsigned *count_out)
 709 {
 710     notmuch_database_t *notmuch = query->notmuch;
 711     Xapian::doccount count = 0;
 712     notmuch_status_t status;
 713
 714     status = _notmuch_query_ensure_parsed (query);
 715     if (status)
 716         return status;
 717
 718     try {
 719         Xapian::Enquire enquire (*notmuch->xapian_db);
 720         Xapian::Query mail_query (talloc_asprintf (query, "%s%s",
 721                                                    _find_prefix ("type"),
 722                                                    type));
 723         Xapian::Query final_query, exclude_query;
 724         Xapian::MSet mset;
 725
 726         final_query = Xapian::Query (Xapian::Query::OP_AND,
 727                                      mail_query, query->xapian_query);
 728
 729         exclude_query = _notmuch_exclude_tags (query);
 730
 731         final_query = Xapian::Query (Xapian::Query::OP_AND_NOT,
 732                                      final_query, exclude_query);
 733
 734         enquire.set_weighting_scheme (Xapian::BoolWeight ());
 735         enquire.set_docid_order (Xapian::Enquire::ASCENDING);
 736
 737         if (_debug_query ()) {
 738             fprintf (stderr, "Exclude query is:\n%s\n",
 739                      exclude_query.get_description ().c_str ());
 740             fprintf (stderr, "Final query is:\n%s\n",
 741                      final_query.get_description ().c_str ());
 742         }
 743
 744         enquire.set_query (final_query);
 745
 746         /*
 747          * Set the checkatleast parameter to the number of documents
 748          * in the database to make get_matches_estimated() exact.
 749          * Set the max parameter to 1 to avoid fetching documents we will discard.
 750          */
 751         mset = enquire.get_mset (0, 1,
 752                                  notmuch->xapian_db->get_doccount ());
 753
 754         count = mset.get_matches_estimated ();
 755
 756     } catch (const Xapian::Error &error) {
 757         _notmuch_database_log (notmuch,
 758                                "A Xapian exception occurred performing query: %s\n",
 759                                error.get_msg ().c_str ());
 760         _notmuch_database_log_append (notmuch,
 761                                       "Query string was: %s\n",
 762                                       query->query_string);
 763         return NOTMUCH_STATUS_XAPIAN_EXCEPTION;
 764     }
 765
 766     *count_out = count;
 767     return NOTMUCH_STATUS_SUCCESS;
 768 }
 769
 770 notmuch_status_t
 771 notmuch_query_count_threads_st (notmuch_query_t *query, unsigned *count)
 772 {
 773     return notmuch_query_count_threads (query, count);
 774 }
 775
 776 notmuch_status_t
 777 notmuch_query_count_threads (notmuch_query_t *query, unsigned *count)
 778 {
 779     notmuch_messages_t *messages;
 780     GHashTable *hash;
 781     notmuch_sort_t sort;
 782     notmuch_status_t ret = NOTMUCH_STATUS_SUCCESS;
 783
 784     sort = query->sort;
 785     query->sort = NOTMUCH_SORT_UNSORTED;
 786     ret = notmuch_query_search_messages (query, &messages);
 787     if (ret)
 788         return ret;
 789     query->sort = sort;
 790     if (messages == NULL)
 791         return NOTMUCH_STATUS_XAPIAN_EXCEPTION;
 792
 793     hash = g_hash_table_new_full (g_str_hash, g_str_equal, NULL, NULL);
 794     if (hash == NULL) {
 795         talloc_free (messages);
 796         return NOTMUCH_STATUS_OUT_OF_MEMORY;
 797     }
 798
 799     while (notmuch_messages_valid (messages)) {
 800         notmuch_message_t *message = notmuch_messages_get (messages);
 801         const char *thread_id = notmuch_message_get_thread_id (message);
 802         char *thread_id_copy = talloc_strdup (messages, thread_id);
 803         if (unlikely (thread_id_copy == NULL)) {
 804             notmuch_message_destroy (message);
 805             ret = NOTMUCH_STATUS_OUT_OF_MEMORY;
 806             goto DONE;
 807         }
 808         g_hash_table_insert (hash, thread_id_copy, NULL);
 809         notmuch_message_destroy (message);
 810         notmuch_messages_move_to_next (messages);
 811     }
 812
 813     *count = g_hash_table_size (hash);
 814
 815   DONE:
 816     g_hash_table_unref (hash);
 817     talloc_free (messages);
 818
 819     return ret;
 820 }
 821
 822 notmuch_database_t *
 823 notmuch_query_get_database (const notmuch_query_t *query)
 824 {
 825     return query->notmuch;
 826 }
 827
 828 notmuch_status_t
 829 _notmuch_query_expand (notmuch_database_t *notmuch, const char *field, Xapian::Query subquery,
 830                        Xapian::Query &output, std::string &msg)
 831 {
 832     std::set<std::string> terms;
 833     const std::string term_prefix =  _find_prefix (field);
 834
 835     if (_debug_query ()) {
 836         fprintf (stderr, "Expanding subquery:\n%s\n",
 837                  subquery.get_description ().c_str ());
 838     }
 839
 840     try {
 841         Xapian::Enquire enquire (*notmuch->xapian_db);
 842         Xapian::MSet mset;
 843
 844         enquire.set_weighting_scheme (Xapian::BoolWeight ());
 845         enquire.set_query (subquery);
 846
 847         mset = enquire.get_mset (0, notmuch->xapian_db->get_doccount ());
 848
 849         for (Xapian::MSetIterator iterator = mset.begin (); iterator != mset.end (); iterator++) {
 850             Xapian::docid doc_id = *iterator;
 851             Xapian::Document doc = notmuch->xapian_db->get_document (doc_id);
 852             Xapian::TermIterator i = doc.termlist_begin ();
 853
 854             for (i.skip_to (term_prefix);
 855                  i != doc.termlist_end () && ((*i).rfind (term_prefix, 0) == 0); i++) {
 856                 terms.insert (*i);
 857             }
 858         }
 859         output = Xapian::Query (Xapian::Query::OP_OR, terms.begin (), terms.end ());
 860         if (_debug_query ()) {
 861             fprintf (stderr, "Expanded query:\n%s\n",
 862                      subquery.get_description ().c_str ());
 863         }
 864
 865     } catch (const Xapian::Error &error) {
 866         _notmuch_database_log (notmuch,
 867                                "A Xapian exception occurred expanding query: %s\n",
 868                                error.get_msg ().c_str ());
 869         msg = error.get_msg ();
 870         return NOTMUCH_STATUS_XAPIAN_EXCEPTION;
 871     }
 872
 873     return NOTMUCH_STATUS_SUCCESS;
 874 }