Fix "too many open files" bug by closing message files when done with them.

The message file header parsing code parses only enough of the file to
find the desired header fields, then it leaves the file open until the
next header parsing call or when the message is no longer in use. If a
large number of messages end up being active, this will quickly run
out of file descriptors.

Here, we add support to explicitly close the message file within a
message, (_notmuch_message_close) and call that from thread
construction code.

Signed-off-by: Keith Packard <keithp@keithp.com>
Edited-by: Carl Worth <cworth@cworth.org>:
Many portions of Keith's original patch have since been solved other
ways, (such as the code that changed the handling of the In-Reply-To
header). So the final version is clean enough that I think even Keith
would be happy to have his name on it.

diff --git a/lib/message.cc b/lib/message.cc
index cb9026439b75647486e4e2b3953da53a8b924e2f..5f8db50fb78b3df8157761387ee30cce4c83f76d 100644 (file)
--- a/lib/message.cc
+++ b/lib/message.cc
@@ -547,6 +547,19 @@ _notmuch_message_sync (notmuch_message_t *message)
     db->replace_document (message->doc_id, message->doc);
 }
 
     db->replace_document (message->doc_id, message->doc);
 }
 

+/* Ensure that 'message' is not holding any file object open. Future
+ * calls to various functions will still automatically open the
+ * message file as needed.
+ */
+void
+_notmuch_message_close (notmuch_message_t *message)
+{
+    if (message->message_file) {
+       notmuch_message_file_close (message->message_file);
+       message->message_file = NULL;
+    }
+}
+

 /* Add a name:value term to 'message', (the actual term will be
  * encoded by prefixing the value with a short prefix). See
  * NORMAL_PREFIX and BOOLEAN_PREFIX arrays for the mapping of term
 /* Add a name:value term to 'message', (the actual term will be
  * encoded by prefixing the value with a short prefix). See
  * NORMAL_PREFIX and BOOLEAN_PREFIX arrays for the mapping of term

diff --git a/lib/notmuch-private.h b/lib/notmuch-private.h
index c398c80add1f840d492a254aed53238b7e97d5af..ddf0674aa9adc61cd61ec43fa4557808b9100589 100644 (file)
--- a/lib/notmuch-private.h
+++ b/lib/notmuch-private.h
@@ -207,6 +207,9 @@ _notmuch_message_set_date (notmuch_message_t *message,
 void
 _notmuch_message_sync (notmuch_message_t *message);
 
 void
 _notmuch_message_sync (notmuch_message_t *message);
 

+void
+_notmuch_message_close (notmuch_message_t *message);
+

 /* index.cc */
 
 notmuch_status_t
 /* index.cc */
 
 notmuch_status_t

diff --git a/lib/thread.cc b/lib/thread.cc
index e38a5b2e0abc336bd16fe03de8cb82902abd984d..baa9e7e84e78ee082703447a0b2cb0386bb89ea8 100644 (file)
--- a/lib/thread.cc
+++ b/lib/thread.cc
@@ -207,6 +207,7 @@ _notmuch_thread_create (void *ctx,
     const char *thread_id_query_string, *matched_query_string;
     notmuch_query_t *thread_id_query, *matched_query;
     notmuch_messages_t *messages;
     const char *thread_id_query_string, *matched_query_string;
     notmuch_query_t *thread_id_query, *matched_query;
     notmuch_messages_t *messages;

+    notmuch_message_t *message;

 
     thread_id_query_string = talloc_asprintf (ctx, "thread:%s", thread_id);
     if (unlikely (query_string == NULL))
 
     thread_id_query_string = talloc_asprintf (ctx, "thread:%s", thread_id);
     if (unlikely (query_string == NULL))


@@ -263,7 +264,9 @@ _notmuch_thread_create (void *ctx,
         notmuch_messages_has_more (messages);
         notmuch_messages_advance (messages))
     {
         notmuch_messages_has_more (messages);
         notmuch_messages_advance (messages))
     {

-       _thread_add_message (thread, notmuch_messages_get (messages));
+       message = notmuch_messages_get (messages);
+       _thread_add_message (thread, message);
+       _notmuch_message_close (message);

     }
 
     notmuch_query_destroy (thread_id_query);
     }
 
     notmuch_query_destroy (thread_id_query);


@@ -272,7 +275,9 @@ _notmuch_thread_create (void *ctx,
         notmuch_messages_has_more (messages);
         notmuch_messages_advance (messages))
     {
         notmuch_messages_has_more (messages);
         notmuch_messages_advance (messages))
     {

-       _thread_add_matched_message (thread, notmuch_messages_get (messages));
+       message = notmuch_messages_get (messages);
+       _thread_add_matched_message (thread, message);
+       _notmuch_message_close (message);

     }
 
     notmuch_query_destroy (matched_query);
     }
 
     notmuch_query_destroy (matched_query);