Please include mutt maildir header cache into current cvs
Hi Thomas,
I think that the maildir header cache is ready for inclusion into the
current cvs. I fixed all issues on my todo list and tested it very hard.
Also with havy loads and locking issues. Also via NFS and so on. Also
you can have status quo ante if you don't give configure an additional
--configur-hcache . Please look at the code and include it in the
current distribution, if you think, that it is ready.
The hcache.c and it's interface is written so that it should be very
easy to use this code to cache all types of headers.
Greetings,
Thomas
References:
http://wwwcip.informatik.uni-erlangen.de/~sithglan/mutt/
http://wwwcip.informatik.uni-erlangen.de/~sithglan/mutt/mutt-cvs-maildir-header-cache.4
--
Thomas Glanzmann ++49 (0) 9131 85-27943 Department of Computer Science IV
Martensstrasse 1 D-91058 Erlangen Germany University of Erlangen-Nuremberg
http://wwwcip.informatik.uni-erlangen.de/
diff -Nru a/Makefile.am b/Makefile.am
--- a/Makefile.am Mon Feb 9 19:19:48 2004
+++ b/Makefile.am Mon Feb 9 19:19:48 2004
@@ -62,7 +62,7 @@
mutt_tunnel.c pop.c pop_auth.c pop_lib.c smime.c pgp.c pgpinvoke.c
pgpkey.c \
pgplib.c sha1.c pgpmicalg.c gnupgparse.c resize.c dotlock.c remailer.c \
browser.h mbyte.h remailer.h url.h mutt_ssl_nss.c \
- pgppacket.c mutt_idna.h
+ pgppacket.c mutt_idna.h hcache.c
EXTRA_DIST = COPYRIGHT GPL OPS OPS.PGP OPS.CRYPT OPS.SMIME TODO \
configure acconfig.h account.h \
diff -Nru a/configure.in b/configure.in
--- a/configure.in Mon Feb 9 19:19:48 2004
+++ b/configure.in Mon Feb 9 19:19:48 2004
@@ -768,6 +768,22 @@
fi])
+dnl -- start cache --
+AC_ARG_ENABLE(hcache, [ --enable-hcache Enable header caching for
Maildir folders],
+[if test x$enableval = xyes; then
+ AC_DEFINE(USE_HCACHE, 1, [Enable header caching for Maildir style
mailboxes])
+ MUTT_LIB_OBJECTS="$MUTT_LIB_OBJECTS hcache.o"
+ LIBS="$LIBS -lgdbm"
+ AC_CACHE_CHECK(for gdbm_open, ac_cv_gdbmopen,
+ [ac_cv_gdbmopen=no
+ AC_TRY_LINK([#include
<gdbm.h>],[gdbm_open(0,0,0,0,0);],[ac_cv_gdbmopen=yes])])
+
+ if test $ac_cv_gdbmopen = no; then
+ AC_MSG_ERROR(You must install libgdbm with --enable-hcache)
+ fi
+fi])
+dnl -- end cache --
+
AC_SUBST(MUTTLIBS)
AC_SUBST(MUTT_LIB_OBJECTS)
AC_SUBST(LIBIMAP)
diff -Nru a/globals.h b/globals.h
--- a/globals.h Mon Feb 9 19:19:48 2004
+++ b/globals.h Mon Feb 9 19:19:48 2004
@@ -63,6 +63,9 @@
WHERE char *Locale;
WHERE char *MailcapPath;
WHERE char *Maildir;
+#if USE_HCACHE
+WHERE char *MaildirCache;
+#endif
WHERE char *MhFlagged;
WHERE char *MhReplied;
WHERE char *MhUnseen;
diff -Nru a/hcache.c b/hcache.c
--- /dev/null Wed Dec 31 16:00:00 1969
+++ b/hcache.c Mon Feb 9 19:19:48 2004
@@ -0,0 +1,417 @@
+/*
+ * Copyright (C) 2004 Thomas Glanzmann <sithglan@xxxxxxxxxxxxxxxxxxxx>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111, USA.
+ */
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif /* HAVE_CONFIG_H */
+
+#include <gdbm.h>
+#include <fcntl.h>
+#include "mutt.h"
+#include "mime.h"
+#include "mx.h"
+#include "lib.h"
+
+static unsigned char *
+dump_int(unsigned int i, unsigned char *d, unsigned int *off)
+{
+ safe_realloc(&d, *off + sizeof(int));
+ memcpy(d + *off, &i, sizeof(int));
+ (*off) += sizeof(int);
+
+ return d;
+}
+
+static void
+restore_int(unsigned int *i, unsigned char *d, unsigned int *off)
+{
+ memcpy(i, d + *off, sizeof(int));
+ (*off) += sizeof(int);
+}
+
+static unsigned char *
+dump_char(char *c, unsigned char *d, unsigned int *off)
+{
+ unsigned int size;
+
+ if (c == NULL) {
+ size = 0;
+ d = dump_int(size, d, off);
+ return d;
+ }
+
+ size = strlen(c) + 1;
+ d = dump_int(size, d, off);
+ safe_realloc(&d, *off + size);
+ memcpy(d + *off, c, size);
+ *off += size;
+
+ return d;
+}
+
+static void
+restore_char(char **c, unsigned char *d, unsigned int *off)
+{
+ unsigned int size;
+ restore_int(&size, d, off);
+
+ if (size == 0) {
+ *c = NULL;
+ return;
+ }
+
+ *c = safe_malloc(size);
+ memcpy(*c, d + *off, size);
+ *off += size;
+}
+
+static void
+skip_char(unsigned char *d, unsigned int *off)
+{
+ unsigned int size;
+ restore_int(&size, d, off);
+ *off += size;
+}
+
+static unsigned char *
+dump_address(ADDRESS *a, unsigned char *d, unsigned int *off)
+{
+ unsigned int counter = 0;
+ unsigned int start_off = *off;
+
+ d = dump_int(0xdeadbeaf, d, off);
+
+ while (a) {
+#ifdef EXACT_ADDRESS
+ d = dump_char(a->val, d, off);
+#endif
+ d = dump_char(a->personal, d, off);
+ d = dump_char(a->mailbox, d, off);
+ d = dump_int(a->group, d, off);
+ a = a->next;
+ counter++;
+ }
+
+ memcpy(d + start_off, &counter, sizeof(int));
+
+ return d;
+}
+
+static void
+restore_address(ADDRESS **a, unsigned char *d, unsigned int *off)
+{
+ unsigned int counter;
+
+ restore_int(&counter, d, off);
+
+ while (counter) {
+ *a = safe_malloc(sizeof(ADDRESS));
+#ifdef EXACT_ADDRESS
+ restore_char(&(*a)->val, d, off);
+#endif
+ restore_char(&(*a)->personal, d, off);
+ restore_char(&(*a)->mailbox, d, off);
+ restore_int((unsigned int *)&(*a)->group, d, off);
+ a = &(*a)->next;
+ counter--;
+ }
+
+ *a = NULL;
+ return;
+}
+
+static unsigned char *
+dump_list(LIST *l, unsigned char *d, unsigned int *off)
+{
+ unsigned int counter = 0;
+ unsigned int start_off = *off;
+
+ d = dump_int(0xdeadbeaf, d, off);
+
+ while (l) {
+ d = dump_char(l->data, d, off);
+ l = l->next;
+ counter++;
+ }
+
+ memcpy(d + start_off, &counter, sizeof(int));
+
+ return d;
+}
+
+static void
+restore_list(LIST **l, unsigned char *d, unsigned int *off)
+{
+ unsigned int counter;
+
+ restore_int(&counter, d, off);
+
+ while (counter) {
+ *l = safe_malloc(sizeof(LIST));
+ restore_char(&(*l)->data, d, off);
+ l = &(*l)->next;
+ counter--;
+ }
+
+ *l = NULL;
+ return;
+}
+
+static unsigned char *
+dump_parameter(PARAMETER *p, unsigned char *d, unsigned int *off)
+{
+ unsigned int counter = 0;
+ unsigned int start_off = *off;
+
+ d = dump_int(0xdeadbeaf, d, off);
+
+ while (p) {
+ d = dump_char(p->attribute, d, off);
+ d = dump_char(p->value, d, off);
+ p = p->next;
+ counter++;
+ }
+
+ memcpy(d + start_off, &counter, sizeof(int));
+
+ return d;
+}
+
+static void
+restore_parameter(PARAMETER **p, unsigned char *d, unsigned int *off)
+{
+ unsigned int counter;
+
+ restore_int(&counter, d, off);
+
+ while (counter) {
+ *p = safe_malloc(sizeof(PARAMETER));
+ restore_char(&(*p)->attribute, d, off);
+ restore_char(&(*p)->value, d, off);
+ p = &(*p)->next;
+ counter--;
+ }
+
+ *p = NULL;
+ return;
+}
+
+static unsigned char *
+dump_body(BODY *c, unsigned char *d, unsigned int *off)
+{
+ safe_realloc(&d, *off + sizeof(BODY));
+ memcpy(d + *off, c, sizeof(BODY));
+ *off += sizeof(BODY);
+
+ d = dump_char(c->xtype, d, off);
+ d = dump_char(c->subtype, d, off);
+
+ d = dump_parameter(c->parameter, d, off);
+
+ d = dump_char(c->description, d, off);
+ d = dump_char(c->form_name, d, off);
+ d = dump_char(c->filename, d, off);
+ d = dump_char(c->d_filename, d, off);
+
+ return d;
+}
+
+static void
+restore_body(BODY *c, unsigned char *d, unsigned int *off)
+{
+ memcpy(c, d + *off, sizeof(BODY));
+ *off += sizeof(BODY);
+
+ restore_char(& c->xtype, d, off);
+ restore_char(& c->subtype, d, off);
+
+ restore_parameter(& c->parameter, d, off);
+
+ restore_char(& c->description, d, off);
+ restore_char(& c->form_name, d, off);
+ restore_char(& c->filename, d, off);
+ restore_char(& c->d_filename, d, off);
+}
+
+static unsigned char *
+dump_envelope(ENVELOPE *e, unsigned char *d, unsigned int *off)
+{
+ d = dump_address(e->return_path, d, off);
+ d = dump_address(e->from, d, off);
+ d = dump_address(e->to, d, off);
+ d = dump_address(e->cc, d, off);
+ d = dump_address(e->bcc, d, off);
+ d = dump_address(e->sender, d, off);
+ d = dump_address(e->reply_to, d, off);
+ d = dump_address(e->mail_followup_to, d, off);
+
+ d = dump_char(e->subject, d, off);
+ d = dump_char(e->real_subj, d, off);
+ d = dump_char(e->message_id, d, off);
+ d = dump_char(e->supersedes, d, off);
+ d = dump_char(e->date, d, off);
+ d = dump_char(e->x_label, d, off);
+
+ d = dump_list(e->references, d, off);
+ d = dump_list(e->in_reply_to, d, off);
+ d = dump_list(e->userhdrs, d, off);
+
+ return d;
+}
+
+static void
+restore_envelope(ENVELOPE *e, unsigned char *d, unsigned int *off)
+{
+ restore_address(& e->return_path, d, off);
+ restore_address(& e->from, d, off);
+ restore_address(& e->to, d, off);
+ restore_address(& e->cc, d, off);
+ restore_address(& e->bcc, d, off);
+ restore_address(& e->sender, d, off);
+ restore_address(& e->reply_to, d, off);
+ restore_address(& e->mail_followup_to, d, off);
+
+ restore_char(& e->subject, d, off);
+ restore_char(& e->real_subj, d, off);
+ restore_char(& e->message_id, d, off);
+ restore_char(& e->supersedes, d, off);
+ restore_char(& e->date, d, off);
+ restore_char(& e->x_label, d, off);
+
+ restore_list(& e->references, d, off);
+ restore_list(& e->in_reply_to, d, off);
+ restore_list(& e->userhdrs, d, off);
+}
+
+
+/* This function transforms a header into a char so that it is useable by
+ * gdbm_store */
+
+#if HAVE_LANGINFO_CODESET
+int
+mutt_hcache_charset_matches(unsigned char *d)
+{
+ unsigned int off = sizeof(struct timeval);
+ char *charset = NULL;
+
+ restore_char(&charset, d, &off);
+
+ return (0 == mutt_strcmp(charset, Charset));
+}
+#endif /* HAVE_LANGINFO_CODESET */
+
+void *
+mutt_hcache_dump(HEADER *h, unsigned int *off)
+{
+ unsigned char *d = NULL;
+ *off = 0;
+ struct timeval now;
+
+ d = safe_malloc(sizeof(struct timeval));
+ gettimeofday(&now, NULL);
+ memcpy(d, &now, sizeof(struct timeval));
+ *off += sizeof(struct timeval);
+
+#if HAVE_LANGINFO_CODESET
+ d = dump_char(Charset, d, off);
+#endif /* HAVE_LANGINFO_CODESET */
+
+
+ safe_realloc(&d, *off + sizeof(HEADER));
+ memcpy(d + *off, h, sizeof(HEADER));
+ *off += sizeof(HEADER);
+
+ d = dump_envelope(h->env, d, off);
+ d = dump_body(h->content, d, off);
+
+ return d;
+}
+
+HEADER *
+mutt_hcache_restore(unsigned char *d, HEADER **oh)
+{
+ unsigned int off = 0;
+ HEADER *h = mutt_new_header();
+
+ /* skip timeval */
+ off += sizeof(struct timeval);
+
+#if HAVE_LANGINFO_CODESET
+ skip_char(d, &off);
+#endif /* HAVE_LANGINFO_CODESET */
+
+ memcpy(h, d + off, sizeof(HEADER));
+ off += sizeof(HEADER);
+
+ h->env = mutt_new_envelope();
+ restore_envelope(h->env, d, &off);
+
+ h->content = mutt_new_body();
+ restore_body(h->content, d, &off);
+
+ h->old = (*oh)->old;
+ h->path = safe_strdup((*oh)->path);
+ mutt_free_header (oh);
+
+ return h;
+}
+
+GDBM_FILE
+mutt_hcache_open(char *path)
+{
+ GDBM_FILE db = NULL;
+
+ if (! path || path[0] == '\0') {
+ return NULL;
+ }
+
+ db = gdbm_open(path, 0, GDBM_WRCREAT, 00600, NULL);
+ if (db) {
+ return db;
+ }
+
+ /* if rw failed try ro */
+ return gdbm_open(path, 0, GDBM_READER, 00600, NULL);
+}
+
+void
+mutt_hcache_close(GDBM_FILE db)
+{
+ if (db) {
+ gdbm_close(db);
+ }
+}
+
+datum
+mutt_hcache_fetch(GDBM_FILE db, datum key)
+{
+ if (! db) {
+ datum ret = {NULL, 0};
+ return ret;
+ }
+ return gdbm_fetch(db, key);
+}
+
+int
+mutt_hcache_store(GDBM_FILE db, datum key, datum data)
+{
+ if (! db) {
+ return -1;
+ }
+ return gdbm_store(db, key, data, GDBM_REPLACE);
+}
diff -Nru a/init.h b/init.h
--- a/init.h Mon Feb 9 19:19:48 2004
+++ b/init.h Mon Feb 9 19:19:48 2004
@@ -981,6 +981,13 @@
** \fBDON'T CHANGE THIS SETTING UNLESS YOU ARE REALLY SURE WHAT YOU ARE
** DOING!\fP
*/
+#if USE_HCACHE
+ { "maildir_cache", DT_PATH, R_NONE, UL &MaildirCache, 0 },
+ /*
+ ** .pp
+ ** Path to the maildir cache file. If unset no cache will be used.
+ */
+#endif /* USE_HCACHE */
{ "maildir_trash", DT_BOOL, R_NONE, OPTMAILDIRTRASH, 0 },
/*
** .pp
diff -Nru a/main.c b/main.c
--- a/main.c Mon Feb 9 19:19:48 2004
+++ b/main.c Mon Feb 9 19:19:48 2004
@@ -411,6 +411,12 @@
"-HAVE_GETADDRINFO "
#endif
+#if USE_HCACHE
+ "+USE_HCACHE "
+#else
+ "-USE_HCACHE "
+#endif
+
);
#ifdef ISPELL
diff -Nru a/mh.c b/mh.c
--- a/mh.c Mon Feb 9 19:19:48 2004
+++ b/mh.c Mon Feb 9 19:19:48 2004
@@ -42,6 +42,10 @@
#include <string.h>
#include <utime.h>
+#if USE_HCACHE
+#include <gdbm.h>
+#endif /* USE_HCACHE */
+
struct maildir
{
HEADER *h;
@@ -779,12 +783,75 @@
return r;
}
-
+#if USE_HCACHE
/*
* This function does the second parsing pass for a maildir-style
* folder.
*/
+static size_t
+maildir_cache_keylen(const char *fn)
+{
+ const char * p = strchr (fn, ':');
+ return p ? (size_t) (p - fn) : strlen (fn);
+}
+
+void maildir_delayed_parsing (CONTEXT * ctx, struct maildir *md)
+{
+ struct maildir *p;
+ GDBM_FILE db = NULL;
+ char fn[_POSIX_PATH_MAX];
+ datum key;
+ datum data;
+ unsigned int size;
+ struct timeval *when = NULL;
+ struct stat lastchanged;
+
+ db = mutt_hcache_open(MaildirCache);
+
+ for (p = md; p; p = p->next) {
+ if (! (p && p->h && !p->header_parsed)) {
+ continue;
+ }
+
+ snprintf(fn, sizeof(fn), "%s/%s", ctx->path, p->h->path + 4);
+ key.dptr = fn;
+ key.dsize = maildir_cache_keylen(fn);
+ data = mutt_hcache_fetch(db, key);
+ when = (struct timeval *) data.dptr;
+
+ snprintf(fn, sizeof (fn), "%s/%s", ctx->path, p->h->path);
+ stat(fn, &lastchanged);
+
+ if (data.dptr != NULL
+ && lastchanged.st_mtime <= when->tv_sec
+#if HAVE_LANGINFO_CODESET
+ && mutt_hcache_charset_matches(data.dptr)
+#endif /* HAVE_LANGINFO_CODESET */
+ ) {
+ p->h = mutt_hcache_restore((unsigned char *)data.dptr,
&p->h);
+ FREE(& data.dptr);
+ maildir_parse_flags(p->h, fn);
+
+ } else if (maildir_parse_message (ctx->magic, fn, p->h->old,
p->h)) {
+ maildir_parse_flags(p->h, fn);
+ p->header_parsed = 1;
+ if (db) {
+ /* only try this if db connection is available
*/
+ data.dptr = mutt_hcache_dump(p->h, &size);
+ data.dsize = size;
+ mutt_hcache_store(db, key, data);
+ FREE(& data.dptr);
+ }
+ } else {
+ mutt_free_header (&p->h);
+ }
+ }
+ mutt_hcache_close(db);
+}
+
+#else /* USE_HCACHE */
+
void maildir_delayed_parsing (CONTEXT * ctx, struct maildir *md)
{
struct maildir *p;
@@ -801,7 +868,7 @@
}
}
-
+#endif /* USE_HCACHE */
/* Read a MH/maildir style mailbox.
*
diff -Nru a/protos.h b/protos.h
--- a/protos.h Mon Feb 9 19:19:48 2004
+++ b/protos.h Mon Feb 9 19:19:48 2004
@@ -99,6 +99,18 @@
ENVELOPE *mutt_read_rfc822_header (FILE *, HEADER *, short, short);
HEADER *mutt_dup_header (HEADER *);
+#if USE_HCACHE
+#include <gdbm.h>
+GDBM_FILE mutt_hcache_open(char *path);
+void mutt_hcache_close(GDBM_FILE db);
+void * mutt_hcache_dump(HEADER *h, unsigned int *off);
+HEADER * mutt_hcache_restore(unsigned char *d, HEADER **oh);
+datum mutt_hcache_fetch(GDBM_FILE db, datum key);
+int mutt_hcache_store(GDBM_FILE db, datum key, datum data);
+void mutt_hcache_restore_charset(char **c, unsigned char *d);
+#endif /* USE_HCACHE */
+
+
ATTACHPTR **mutt_gen_attach_list (BODY *, int, ATTACHPTR **, short *, short *,
int, int);
time_t mutt_decrease_mtime (const char *, struct stat *);