This was for some time testing, (to see how fast xapian could be
if we were strictly adding documents and not doing any other IO
or computation). The answer is that xapian is quite fast, (on
the order of 1000 documents per second).
+unsigned int MAX_TERMS = 0;
+
+static void
+print_escaped_string (const char *s)
+{
+ printf ("\"");
+
+ while (*s) {
+ if (*s == '"')
+ printf ("\\");
+ printf ("%c", *s);
+ s++;
+ }
+
+ printf ("\"");
+}
+
static void
print_document_terms (Xapian::Document doc)
{
static void
print_document_terms (Xapian::Document doc)
{
- Xapian::TermIterator i;
+ Xapian::TermIterator it;
+ unsigned int i;
+
+ printf (" {\n");
+
+ for (it = doc.termlist_begin (), i = 0;
+ it != doc.termlist_end ();
+ it++, i++)
+ {
+ printf (" ");
+ print_escaped_string ((*it).c_str());
+ printf (",\n");
+ }
+ for ( ; i < MAX_TERMS; i++)
+ printf (" \"\",\n");
- for (i = doc.termlist_begin (); i != doc.termlist_end (); i++)
- cout << "\t" << *i << endl;
int value_no, value_int;
double value_float;
int value_no, value_int;
double value_float;
- printf (" Values:\n");
-
for (i = doc.values_begin (); i != doc.values_end (); i++) {
value_no = i.get_valueno();
for (i = doc.values_begin (); i != doc.values_end (); i++) {
value_no = i.get_valueno();
- cout << "\t" << i.get_valueno() << ": ";
if (vector_int_contains (UNSERIALIZE, value_no)) {
value_float = Xapian::sortable_unserialise (*i);
value_int = value_float;
if (value_int == value_float)
if (vector_int_contains (UNSERIALIZE, value_no)) {
value_float = Xapian::sortable_unserialise (*i);
value_int = value_float;
if (value_int == value_float)
+ printf ("%d", value_int);
+ printf ("\"%f\"", value_float);
+ print_escaped_string ((*i).c_str ());
- printf ("Document %u:\n", id);
doc = db.get_document (id);
doc = db.get_document (id);
- printf (" Data:\n");
- cout << "\t" << doc.get_data () << endl;
+ printf (" \"%s\",\n", doc.get_data ().c_str());
print_document_terms (doc);
print_document_values (doc);
print_document_terms (doc);
print_document_values (doc);
UNSERIALIZE.push_back (atoi (argv[i]));
try {
UNSERIALIZE.push_back (atoi (argv[i]));
try {
Xapian::Database db;
Xapian::PostingIterator i;
Xapian::docid doc_id;
db = Xapian::Database (database_path);
Xapian::Database db;
Xapian::PostingIterator i;
Xapian::docid doc_id;
db = Xapian::Database (database_path);
+
+ for (i = db.postlist_begin (""); i != db.postlist_end (""); i++) {
+ Xapian::Document doc;
+
+ doc_id = *i;
+
+ doc = db.get_document (doc_id);
+
+ if (doc.termlist_count () > MAX_TERMS)
+ MAX_TERMS = doc.termlist_count ();
+ }
+
+ printf ("#define MAX_TERMS %d\n\n", MAX_TERMS);
+
+ printf ("typedef struct {\n"
+ " char data[255];\n"
+ " char terms[MAX_TERMS][255];\n"
+ " char message_id[255];\n"
+ " char thread_id[4096];\n"
+ " time_t time;\n"
+ "} document_dump_t;\n\n");
+
+ printf ("document_dump_t dump[] = {\n");
+
for (i = db.postlist_begin (""); i != db.postlist_end (""); i++) {
doc_id = *i;
print_document (db, doc_id);
}
for (i = db.postlist_begin (""); i != db.postlist_end (""); i++) {
doc_id = *i;
print_document (db, doc_id);
}
} catch (const Xapian::Error &error) {
cerr << "A Xapian exception occurred: " << error.get_msg () << endl;
exit (1);
} catch (const Xapian::Error &error) {
cerr << "A Xapian exception occurred: " << error.get_msg () << endl;
exit (1);