<<< Date Index >>>     <<< Thread Index >>>

Re: [PATCH] hcache reorganization



Hi,

* Brendan Cully wrote:

I agree that 20 is too low. I'd like to see this unified with
$message_cache_clean ($cache_clean perhaps?) and default to off.

Good idea, patch attached. I'm not totally happy with the name cache_clean. I think something like $cache_maintain fits a little better since it cleans/wipes only stale entries which is more a maintenance task than a pure cleaning task...

The
disadvantage of that setting is that you have to visit each folder you
want cleaned up while the setting is active. But the advantage is that
you don't pay the price for the garbage collection for any folder
where you aren't bothered by its space usage.

Exactly. For the disadvantage I've been thinking about a command line option like --cache-reorg or similar which would read the config and open all folders defined by mailboxes that could possibly have a hcache. That would need to temporarily enable the setting.

I once had a patch for that which I used everytime the hcache structure changed and which did rebuild all caches from scratch when the machine was idle so that I wouldn't have to wait when opening a random folder later.

I suppose another possibility is to trigger it when a certain ratio of
dead entries is reached.

Basically I agree. However, I think we can't get the ratio right for everybody (like the 250ms delay for progress updates) so that hardcoding it would sooner or later lead to yet another option. And I think we've already enough (if not too much) of those minor tuning options.

Rocco
comparing with ../pdmef/feature/hcache-reorg
searching for changes
diff --git a/hcache.c b/hcache.c
--- a/hcache.c
+++ b/hcache.c
@@ -48,31 +48,23 @@
 #include "lib.h"
 #include "md5.h"
 
-#if HAVE_QDBM
 static struct header_cache
 {
-  VILLA *db;
   char *folder;
   unsigned int crc;
-} HEADER_CACHE;
+#if HAVE_QDBM
+  VILLA *db;
 #elif HAVE_GDBM
-static struct header_cache
-{
   GDBM_FILE db;
-  char *folder;
-  unsigned int crc;
-} HEADER_CACHE;
 #elif HAVE_DB4
-static struct header_cache
-{
   DB_ENV *env;
   DB *db;
-  char *folder;
-  unsigned int crc;
   int fd;
   char lockfile[_POSIX_PATH_MAX];
+#endif
 } HEADER_CACHE;
 
+#if HAVE_DB4
 static void mutt_hcache_dbt_init(DBT * dbt, void *data, size_t len);
 static void mutt_hcache_dbt_empty_init(DBT * dbt);
 #endif
@@ -791,8 +783,7 @@ hcache_open_qdbm (struct header_cache* h
     return -1;
 }
 
-void
-mutt_hcache_close(header_cache_t *h)
+static void hcache_close_qdbm (header_cache_t *h)
 {
   if (!h)
     return;
@@ -820,6 +811,14 @@ mutt_hcache_delete(header_cache_t *h, co
   return vlout(h->db, path, ksize);
 }
 
+
+static int hcache_reorg_qdbm (header_cache_t *h)
+{
+  if (!h)
+    return -1;
+  return vloptimize (h->db);
+}
+
 #elif HAVE_GDBM
 static int
 hcache_open_gdbm (struct header_cache* h, const char* path)
@@ -838,8 +837,7 @@ hcache_open_gdbm (struct header_cache* h
   return -1;
 }
 
-void
-mutt_hcache_close(header_cache_t *h)
+static void hcache_close_gdbm (header_cache_t *h)
 {
   if (!h)
     return;
@@ -867,6 +865,14 @@ mutt_hcache_delete(header_cache_t *h, co
 
   return gdbm_delete(h->db, key);
 }
+
+static int hcache_reorg_gdbm (header_cache_t *h)
+{
+  if (!h)
+    return -1;
+  return gdbm_reorganize (h->db);
+}
+
 #elif HAVE_DB4
 
 static void
@@ -942,8 +948,7 @@ hcache_open_db4 (struct header_cache* h,
   return -1;
 }
 
-void
-mutt_hcache_close(header_cache_t *h)
+static void hcache_close_db4 (header_cache_t *h)
 {
   if (!h)
     return;
@@ -972,6 +977,12 @@ mutt_hcache_delete(header_cache_t *h, co
   mutt_hcache_dbt_init(&key, (void *) filename, keylen(filename));
   return h->db->del(h->db, NULL, &key, 0);
 }
+
+static int hcache_reorg_db4 (header_cache_t *h)
+{
+  return 0;
+}
+
 #endif
 
 header_cache_t *
@@ -1019,6 +1030,30 @@ mutt_hcache_open(const char *path, const
   }
 }
 
+void mutt_hcache_close (header_cache_t *h, int flags)
+{
+  void (*cl) (struct header_cache* h);
+  int (*reorg) (struct header_cache* h);
+
+#if HAVE_QDBM
+  reorg = hcache_reorg_qdbm;
+  cl = hcache_close_qdbm;
+#elif HAVE_GDBM
+  reorg = hcache_reorg_gdbm;
+  cl = hcache_close_gdbm;
+#elif HAVE_DB4
+  reorg = hcache_reorg_db4;
+  cl = hcache_close_db4;
+#endif
+
+  if ((flags & M_HC_REORG) && option (OPTCACHECLEAN))
+  {
+    dprint (4, (debugfile, "hcache: reorganizing db for %s\n", h->folder));
+    reorg (h);
+  }
+  cl (h);
+}
+
 #if HAVE_DB4
 const char *mutt_hcache_backend (void)
 {
diff --git a/hcache.h b/hcache.h
--- a/hcache.h
+++ b/hcache.h
@@ -28,7 +28,11 @@ typedef int (*hcache_namer_t)(const char
 
 header_cache_t *mutt_hcache_open(const char *path, const char *folder,
   hcache_namer_t namer);
-void mutt_hcache_close(header_cache_t *h);
+
+#define M_HC_REORG     (1<<0)          /* reorg/optimize hcache on close */
+
+void mutt_hcache_close (header_cache_t *h, int flags);
+
 HEADER *mutt_hcache_restore(const unsigned char *d, HEADER **oh);
 void *mutt_hcache_fetch(header_cache_t *h, const char *filename, size_t 
(*keylen)(const char *fn));
 void *mutt_hcache_fetch_raw (header_cache_t *h, const char *filename,
diff --git a/imap/imap.c b/imap/imap.c
--- a/imap/imap.c
+++ b/imap/imap.c
@@ -282,7 +282,7 @@ void imap_expunge_mailbox (IMAP_DATA* id
   }
 
 #if USE_HCACHE
-  imap_hcache_close (idata);
+  imap_hcache_close (idata, M_HC_REORG);
 #endif
 
   /* We may be called on to expunge at any time. We can't rely on the caller
@@ -1186,7 +1186,7 @@ int imap_sync_mailbox (CONTEXT* ctx, int
   }
 
 #if USE_HCACHE
-  imap_hcache_close (idata);
+  imap_hcache_close (idata, M_HC_REORG);
 #endif
 
   /* sync +/- flags for the five flags mutt cares about */
@@ -1262,7 +1262,7 @@ int imap_sync_mailbox (CONTEXT* ctx, int
     idata->state = IMAP_AUTHENTICATED;
   }
 
-  if (option (OPTMESSAGECACHECLEAN))
+  if (option (OPTCACHECLEAN))
     imap_cache_clean (idata);
 
   rc = 0;
@@ -1602,7 +1602,7 @@ IMAP_STATUS* imap_mboxcache_get (IMAP_DA
   {
     uidvalidity = mutt_hcache_fetch_raw (hc, "/UIDVALIDITY", 
imap_hcache_keylen);
     uidnext = mutt_hcache_fetch_raw (hc, "/UIDNEXT", imap_hcache_keylen);
-    mutt_hcache_close (hc);
+    mutt_hcache_close (hc, 0);
     if (uidvalidity)
     {
       if (!status)
diff --git a/imap/imap_private.h b/imap/imap_private.h
--- a/imap/imap_private.h
+++ b/imap/imap_private.h
@@ -261,7 +261,7 @@ int imap_cache_clean (IMAP_DATA* idata);
 /* util.c */
 #ifdef USE_HCACHE
 header_cache_t* imap_hcache_open (IMAP_DATA* idata, const char* path);
-void imap_hcache_close (IMAP_DATA* idata);
+void imap_hcache_close (IMAP_DATA* idata, int flags);
 HEADER* imap_hcache_get (IMAP_DATA* idata, unsigned int uid);
 int imap_hcache_put (IMAP_DATA* idata, HEADER* h);
 int imap_hcache_del (IMAP_DATA* idata, unsigned int uid);
diff --git a/imap/message.c b/imap/message.c
--- a/imap/message.c
+++ b/imap/message.c
@@ -208,7 +208,7 @@ int imap_read_headers (IMAP_DATA* idata,
       {
         if (h.data)
           imap_free_header_data ((void**) (void*) &h.data);
-        imap_hcache_close (idata);
+        imap_hcache_close (idata, 0);
         fclose (fp);
         return -1;
       }
@@ -303,7 +303,7 @@ int imap_read_headers (IMAP_DATA* idata,
       if (h.data)
         imap_free_header_data ((void**) (void*) &h.data);
 #if USE_HCACHE
-      imap_hcache_close (idata);
+      imap_hcache_close (idata, 0);
 #endif
       fclose (fp);
       return -1;
@@ -335,7 +335,7 @@ int imap_read_headers (IMAP_DATA* idata,
     mutt_hcache_store_raw (idata->hcache, "/UIDNEXT", &idata->uidnext,
                           sizeof (idata->uidnext), imap_hcache_keylen);
 
-  imap_hcache_close (idata);
+  imap_hcache_close (idata, 0);
 #endif /* USE_HCACHE */
 
   fclose(fp);
diff --git a/imap/util.c b/imap/util.c
--- a/imap/util.c
+++ b/imap/util.c
@@ -101,12 +101,12 @@ header_cache_t* imap_hcache_open (IMAP_D
   return mutt_hcache_open (HeaderCache, cachepath, imap_hcache_namer);
 }
 
-void imap_hcache_close (IMAP_DATA* idata)
+void imap_hcache_close (IMAP_DATA* idata, int flags)
 {
   if (!idata->hcache)
     return;
 
-  mutt_hcache_close (idata->hcache);
+  mutt_hcache_close (idata->hcache, flags);
   idata->hcache = NULL;
 }
 
diff --git a/init.h b/init.h
--- a/init.h
+++ b/init.h
@@ -305,6 +305,22 @@ struct option_t MuttVars[] = {
   ** follow these menus.  The option is disabled by default because many 
   ** visual terminals don't permit making the cursor invisible.
   */
+#if defined(USE_IMAP) || defined(USE_POP)
+  { "message_cache_clean",     DT_SYN,  R_NONE, UL "cache_clean", 0 },
+#endif
+#if defined(USE_IMAP) || defined(USE_POP) || defined(USE_HCACHE)
+  { "cache_clean", DT_BOOL, R_NONE, OPTCACHECLEAN, 0 },
+  /*
+  ** .pp
+  ** If set, mutt will clean out obsolete entries from the message cache
+  ** when the mailbox is synchronized as well as use the header cache
+  ** backend's optimization routines for the database file (if header
+  ** caching is enabled and the backend support optimization).
+  ** .pp
+  ** You probably only want to set it every once in a while, since it
+  ** can be a little slow.
+  */
+#endif
   { "check_mbox_size", DT_BOOL, R_NONE, OPTCHECKMBOXSIZE, 0 },
   /*
   ** .pp
@@ -1335,13 +1351,6 @@ struct option_t MuttVars[] = {
   ** your IMAP and POP servers here. You are free to remove entries at any
   ** time, for instance if stale entries accumulate because you have
   ** deleted messages with another mail client.
-  */
-  { "message_cache_clean", DT_BOOL, R_NONE, OPTMESSAGECACHECLEAN, 0 },
-  /*
-  ** .pp
-  ** If set, mutt will clean out obsolete entries from the cache when
-  ** the mailbox is synchronized. You probably only want to set it
-  ** every once in a while, since it can be a little slow.
   */
 #endif
   { "message_format",  DT_STR,  R_NONE, UL &MsgFmt, UL "%s" },
diff --git a/mh.c b/mh.c
--- a/mh.c
+++ b/mh.c
@@ -1085,7 +1085,7 @@ void maildir_delayed_parsing (CONTEXT * 
     last = p;
    }
 #if USE_HCACHE
-  mutt_hcache_close (hc);
+  mutt_hcache_close (hc, 0);
 #endif
 
   mh_sort_natural (ctx, md);
@@ -1686,7 +1686,7 @@ int mh_sync_mailbox (CONTEXT * ctx, int 
 
 #if USE_HCACHE
   if (ctx->magic == M_MAILDIR || ctx->magic == M_MH)
-    mutt_hcache_close (hc);
+    mutt_hcache_close (hc, M_HC_REORG);
 #endif /* USE_HCACHE */
 
   if (ctx->magic == M_MH)
@@ -1713,7 +1713,7 @@ err:
 err:
 #if USE_HCACHE
   if (ctx->magic == M_MAILDIR || ctx->magic == M_MH)
-    mutt_hcache_close (hc);
+    mutt_hcache_close (hc, M_HC_REORG);
 #endif /* USE_HCACHE */
   return -1;
 }
diff --git a/mutt.h b/mutt.h
--- a/mutt.h
+++ b/mutt.h
@@ -339,6 +339,9 @@ enum
   OPTBEEPNEW,
   OPTBOUNCEDELIVERED,
   OPTBRAILLEFRIENDLY,
+#if defined(USE_IMAP) || defined(USE_POP) || defined(USE_HCACHE)
+  OPTCACHECLEAN,
+#endif
   OPTCHECKMBOXSIZE,
   OPTCHECKNEW,
   OPTCOLLAPSEUNREAD,
@@ -400,9 +403,6 @@ enum
   OPTMARKOLD,
   OPTMENUSCROLL,       /* scroll menu instead of implicit next-page */
   OPTMENUMOVEOFF,      /* allow menu to scroll past last entry */
-#if defined(USE_IMAP) || defined(USE_POP)
-  OPTMESSAGECACHECLEAN,
-#endif
   OPTMETAKEY,          /* interpret ALT-x as ESC-x */
   OPTMETOO,
   OPTMHPURGE,
diff --git a/pop.c b/pop.c
--- a/pop.c
+++ b/pop.c
@@ -354,7 +354,7 @@ static int pop_fetch_headers (CONTEXT *c
   }
 
 #if USE_HCACHE
-    mutt_hcache_close (hc);
+    mutt_hcache_close (hc, 0);
 #endif
 
   if (ret < 0)
@@ -368,7 +368,7 @@ static int pop_fetch_headers (CONTEXT *c
    * clean up cache, i.e. wipe messages deleted outside
    * the availability of our cache
    */
-  if (option (OPTMESSAGECACHECLEAN))
+  if (option (OPTCACHECLEAN))
     mutt_bcache_list (pop_data->bcache, msg_cache_check, (void*)ctx);
 
   mutt_clear_error ();
@@ -668,7 +668,7 @@ int pop_sync_mailbox (CONTEXT *ctx, int 
     }
 
 #if USE_HCACHE
-    mutt_hcache_close (hc);
+    mutt_hcache_close (hc, M_HC_REORG);
 #endif
 
     if (ret == 0)