Re: [PATCH] hcache reorganization
Hi,
* Brendan Cully wrote:
I agree that 20 is too low. I'd like to see this unified with
$message_cache_clean ($cache_clean perhaps?) and default to off.
Good idea, patch attached. I'm not totally happy with the name
cache_clean. I think something like $cache_maintain fits a little better
since it cleans/wipes only stale entries which is more a maintenance
task than a pure cleaning task...
The
disadvantage of that setting is that you have to visit each folder you
want cleaned up while the setting is active. But the advantage is that
you don't pay the price for the garbage collection for any folder
where you aren't bothered by its space usage.
Exactly. For the disadvantage I've been thinking about a command line
option like --cache-reorg or similar which would read the config and
open all folders defined by mailboxes that could possibly have a hcache.
That would need to temporarily enable the setting.
I once had a patch for that which I used everytime the hcache structure
changed and which did rebuild all caches from scratch when the machine
was idle so that I wouldn't have to wait when opening a random folder
later.
I suppose another possibility is to trigger it when a certain ratio of
dead entries is reached.
Basically I agree. However, I think we can't get the ratio right for
everybody (like the 250ms delay for progress updates) so that hardcoding
it would sooner or later lead to yet another option. And I think we've
already enough (if not too much) of those minor tuning options.
Rocco
comparing with ../pdmef/feature/hcache-reorg
searching for changes
diff --git a/hcache.c b/hcache.c
--- a/hcache.c
+++ b/hcache.c
@@ -48,31 +48,23 @@
#include "lib.h"
#include "md5.h"
-#if HAVE_QDBM
static struct header_cache
{
- VILLA *db;
char *folder;
unsigned int crc;
-} HEADER_CACHE;
+#if HAVE_QDBM
+ VILLA *db;
#elif HAVE_GDBM
-static struct header_cache
-{
GDBM_FILE db;
- char *folder;
- unsigned int crc;
-} HEADER_CACHE;
#elif HAVE_DB4
-static struct header_cache
-{
DB_ENV *env;
DB *db;
- char *folder;
- unsigned int crc;
int fd;
char lockfile[_POSIX_PATH_MAX];
+#endif
} HEADER_CACHE;
+#if HAVE_DB4
static void mutt_hcache_dbt_init(DBT * dbt, void *data, size_t len);
static void mutt_hcache_dbt_empty_init(DBT * dbt);
#endif
@@ -791,8 +783,7 @@ hcache_open_qdbm (struct header_cache* h
return -1;
}
-void
-mutt_hcache_close(header_cache_t *h)
+static void hcache_close_qdbm (header_cache_t *h)
{
if (!h)
return;
@@ -820,6 +811,14 @@ mutt_hcache_delete(header_cache_t *h, co
return vlout(h->db, path, ksize);
}
+
+static int hcache_reorg_qdbm (header_cache_t *h)
+{
+ if (!h)
+ return -1;
+ return vloptimize (h->db);
+}
+
#elif HAVE_GDBM
static int
hcache_open_gdbm (struct header_cache* h, const char* path)
@@ -838,8 +837,7 @@ hcache_open_gdbm (struct header_cache* h
return -1;
}
-void
-mutt_hcache_close(header_cache_t *h)
+static void hcache_close_gdbm (header_cache_t *h)
{
if (!h)
return;
@@ -867,6 +865,14 @@ mutt_hcache_delete(header_cache_t *h, co
return gdbm_delete(h->db, key);
}
+
+static int hcache_reorg_gdbm (header_cache_t *h)
+{
+ if (!h)
+ return -1;
+ return gdbm_reorganize (h->db);
+}
+
#elif HAVE_DB4
static void
@@ -942,8 +948,7 @@ hcache_open_db4 (struct header_cache* h,
return -1;
}
-void
-mutt_hcache_close(header_cache_t *h)
+static void hcache_close_db4 (header_cache_t *h)
{
if (!h)
return;
@@ -972,6 +977,12 @@ mutt_hcache_delete(header_cache_t *h, co
mutt_hcache_dbt_init(&key, (void *) filename, keylen(filename));
return h->db->del(h->db, NULL, &key, 0);
}
+
+static int hcache_reorg_db4 (header_cache_t *h)
+{
+ return 0;
+}
+
#endif
header_cache_t *
@@ -1019,6 +1030,30 @@ mutt_hcache_open(const char *path, const
}
}
+void mutt_hcache_close (header_cache_t *h, int flags)
+{
+ void (*cl) (struct header_cache* h);
+ int (*reorg) (struct header_cache* h);
+
+#if HAVE_QDBM
+ reorg = hcache_reorg_qdbm;
+ cl = hcache_close_qdbm;
+#elif HAVE_GDBM
+ reorg = hcache_reorg_gdbm;
+ cl = hcache_close_gdbm;
+#elif HAVE_DB4
+ reorg = hcache_reorg_db4;
+ cl = hcache_close_db4;
+#endif
+
+ if ((flags & M_HC_REORG) && option (OPTCACHECLEAN))
+ {
+ dprint (4, (debugfile, "hcache: reorganizing db for %s\n", h->folder));
+ reorg (h);
+ }
+ cl (h);
+}
+
#if HAVE_DB4
const char *mutt_hcache_backend (void)
{
diff --git a/hcache.h b/hcache.h
--- a/hcache.h
+++ b/hcache.h
@@ -28,7 +28,11 @@ typedef int (*hcache_namer_t)(const char
header_cache_t *mutt_hcache_open(const char *path, const char *folder,
hcache_namer_t namer);
-void mutt_hcache_close(header_cache_t *h);
+
+#define M_HC_REORG (1<<0) /* reorg/optimize hcache on close */
+
+void mutt_hcache_close (header_cache_t *h, int flags);
+
HEADER *mutt_hcache_restore(const unsigned char *d, HEADER **oh);
void *mutt_hcache_fetch(header_cache_t *h, const char *filename, size_t
(*keylen)(const char *fn));
void *mutt_hcache_fetch_raw (header_cache_t *h, const char *filename,
diff --git a/imap/imap.c b/imap/imap.c
--- a/imap/imap.c
+++ b/imap/imap.c
@@ -282,7 +282,7 @@ void imap_expunge_mailbox (IMAP_DATA* id
}
#if USE_HCACHE
- imap_hcache_close (idata);
+ imap_hcache_close (idata, M_HC_REORG);
#endif
/* We may be called on to expunge at any time. We can't rely on the caller
@@ -1186,7 +1186,7 @@ int imap_sync_mailbox (CONTEXT* ctx, int
}
#if USE_HCACHE
- imap_hcache_close (idata);
+ imap_hcache_close (idata, M_HC_REORG);
#endif
/* sync +/- flags for the five flags mutt cares about */
@@ -1262,7 +1262,7 @@ int imap_sync_mailbox (CONTEXT* ctx, int
idata->state = IMAP_AUTHENTICATED;
}
- if (option (OPTMESSAGECACHECLEAN))
+ if (option (OPTCACHECLEAN))
imap_cache_clean (idata);
rc = 0;
@@ -1602,7 +1602,7 @@ IMAP_STATUS* imap_mboxcache_get (IMAP_DA
{
uidvalidity = mutt_hcache_fetch_raw (hc, "/UIDVALIDITY",
imap_hcache_keylen);
uidnext = mutt_hcache_fetch_raw (hc, "/UIDNEXT", imap_hcache_keylen);
- mutt_hcache_close (hc);
+ mutt_hcache_close (hc, 0);
if (uidvalidity)
{
if (!status)
diff --git a/imap/imap_private.h b/imap/imap_private.h
--- a/imap/imap_private.h
+++ b/imap/imap_private.h
@@ -261,7 +261,7 @@ int imap_cache_clean (IMAP_DATA* idata);
/* util.c */
#ifdef USE_HCACHE
header_cache_t* imap_hcache_open (IMAP_DATA* idata, const char* path);
-void imap_hcache_close (IMAP_DATA* idata);
+void imap_hcache_close (IMAP_DATA* idata, int flags);
HEADER* imap_hcache_get (IMAP_DATA* idata, unsigned int uid);
int imap_hcache_put (IMAP_DATA* idata, HEADER* h);
int imap_hcache_del (IMAP_DATA* idata, unsigned int uid);
diff --git a/imap/message.c b/imap/message.c
--- a/imap/message.c
+++ b/imap/message.c
@@ -208,7 +208,7 @@ int imap_read_headers (IMAP_DATA* idata,
{
if (h.data)
imap_free_header_data ((void**) (void*) &h.data);
- imap_hcache_close (idata);
+ imap_hcache_close (idata, 0);
fclose (fp);
return -1;
}
@@ -303,7 +303,7 @@ int imap_read_headers (IMAP_DATA* idata,
if (h.data)
imap_free_header_data ((void**) (void*) &h.data);
#if USE_HCACHE
- imap_hcache_close (idata);
+ imap_hcache_close (idata, 0);
#endif
fclose (fp);
return -1;
@@ -335,7 +335,7 @@ int imap_read_headers (IMAP_DATA* idata,
mutt_hcache_store_raw (idata->hcache, "/UIDNEXT", &idata->uidnext,
sizeof (idata->uidnext), imap_hcache_keylen);
- imap_hcache_close (idata);
+ imap_hcache_close (idata, 0);
#endif /* USE_HCACHE */
fclose(fp);
diff --git a/imap/util.c b/imap/util.c
--- a/imap/util.c
+++ b/imap/util.c
@@ -101,12 +101,12 @@ header_cache_t* imap_hcache_open (IMAP_D
return mutt_hcache_open (HeaderCache, cachepath, imap_hcache_namer);
}
-void imap_hcache_close (IMAP_DATA* idata)
+void imap_hcache_close (IMAP_DATA* idata, int flags)
{
if (!idata->hcache)
return;
- mutt_hcache_close (idata->hcache);
+ mutt_hcache_close (idata->hcache, flags);
idata->hcache = NULL;
}
diff --git a/init.h b/init.h
--- a/init.h
+++ b/init.h
@@ -305,6 +305,22 @@ struct option_t MuttVars[] = {
** follow these menus. The option is disabled by default because many
** visual terminals don't permit making the cursor invisible.
*/
+#if defined(USE_IMAP) || defined(USE_POP)
+ { "message_cache_clean", DT_SYN, R_NONE, UL "cache_clean", 0 },
+#endif
+#if defined(USE_IMAP) || defined(USE_POP) || defined(USE_HCACHE)
+ { "cache_clean", DT_BOOL, R_NONE, OPTCACHECLEAN, 0 },
+ /*
+ ** .pp
+ ** If set, mutt will clean out obsolete entries from the message cache
+ ** when the mailbox is synchronized as well as use the header cache
+ ** backend's optimization routines for the database file (if header
+ ** caching is enabled and the backend support optimization).
+ ** .pp
+ ** You probably only want to set it every once in a while, since it
+ ** can be a little slow.
+ */
+#endif
{ "check_mbox_size", DT_BOOL, R_NONE, OPTCHECKMBOXSIZE, 0 },
/*
** .pp
@@ -1335,13 +1351,6 @@ struct option_t MuttVars[] = {
** your IMAP and POP servers here. You are free to remove entries at any
** time, for instance if stale entries accumulate because you have
** deleted messages with another mail client.
- */
- { "message_cache_clean", DT_BOOL, R_NONE, OPTMESSAGECACHECLEAN, 0 },
- /*
- ** .pp
- ** If set, mutt will clean out obsolete entries from the cache when
- ** the mailbox is synchronized. You probably only want to set it
- ** every once in a while, since it can be a little slow.
*/
#endif
{ "message_format", DT_STR, R_NONE, UL &MsgFmt, UL "%s" },
diff --git a/mh.c b/mh.c
--- a/mh.c
+++ b/mh.c
@@ -1085,7 +1085,7 @@ void maildir_delayed_parsing (CONTEXT *
last = p;
}
#if USE_HCACHE
- mutt_hcache_close (hc);
+ mutt_hcache_close (hc, 0);
#endif
mh_sort_natural (ctx, md);
@@ -1686,7 +1686,7 @@ int mh_sync_mailbox (CONTEXT * ctx, int
#if USE_HCACHE
if (ctx->magic == M_MAILDIR || ctx->magic == M_MH)
- mutt_hcache_close (hc);
+ mutt_hcache_close (hc, M_HC_REORG);
#endif /* USE_HCACHE */
if (ctx->magic == M_MH)
@@ -1713,7 +1713,7 @@ err:
err:
#if USE_HCACHE
if (ctx->magic == M_MAILDIR || ctx->magic == M_MH)
- mutt_hcache_close (hc);
+ mutt_hcache_close (hc, M_HC_REORG);
#endif /* USE_HCACHE */
return -1;
}
diff --git a/mutt.h b/mutt.h
--- a/mutt.h
+++ b/mutt.h
@@ -339,6 +339,9 @@ enum
OPTBEEPNEW,
OPTBOUNCEDELIVERED,
OPTBRAILLEFRIENDLY,
+#if defined(USE_IMAP) || defined(USE_POP) || defined(USE_HCACHE)
+ OPTCACHECLEAN,
+#endif
OPTCHECKMBOXSIZE,
OPTCHECKNEW,
OPTCOLLAPSEUNREAD,
@@ -400,9 +403,6 @@ enum
OPTMARKOLD,
OPTMENUSCROLL, /* scroll menu instead of implicit next-page */
OPTMENUMOVEOFF, /* allow menu to scroll past last entry */
-#if defined(USE_IMAP) || defined(USE_POP)
- OPTMESSAGECACHECLEAN,
-#endif
OPTMETAKEY, /* interpret ALT-x as ESC-x */
OPTMETOO,
OPTMHPURGE,
diff --git a/pop.c b/pop.c
--- a/pop.c
+++ b/pop.c
@@ -354,7 +354,7 @@ static int pop_fetch_headers (CONTEXT *c
}
#if USE_HCACHE
- mutt_hcache_close (hc);
+ mutt_hcache_close (hc, 0);
#endif
if (ret < 0)
@@ -368,7 +368,7 @@ static int pop_fetch_headers (CONTEXT *c
* clean up cache, i.e. wipe messages deleted outside
* the availability of our cache
*/
- if (option (OPTMESSAGECACHECLEAN))
+ if (option (OPTCACHECLEAN))
mutt_bcache_list (pop_data->bcache, msg_cache_check, (void*)ctx);
mutt_clear_error ();
@@ -668,7 +668,7 @@ int pop_sync_mailbox (CONTEXT *ctx, int
}
#if USE_HCACHE
- mutt_hcache_close (hc);
+ mutt_hcache_close (hc, M_HC_REORG);
#endif
if (ret == 0)