[PATCH] support for IMAP COMPRESS extension (RFC4978)
I noticed that Gmail supports RFC4978, so I took a crack at implementing it.
It seems to work pretty well, as the compressor stats report that both sending
and receiving are compressed to about ~30% of the original size.
Consider this experimental, as it is lightly tested.
To use it, you need zlib installed (pretty sure almost everyone who is
compiling has the devel package installed).
Then add "set imap_compress" in your ~/.muttrc to enable it if the server
supports it.
me
# HG changeset patch
# User me@xxxxxxxx
# Date 1281740962 25200
# Branch HEAD
# Node ID 633cd805f2580fc83f24311e8dd395356b74ba23
# Parent 6572e8bcd723a34d9457650fe272ff9633033ccf
support for IMAP COMPRESS extension
diff --git a/Makefile.am b/Makefile.am
--- a/Makefile.am
+++ b/Makefile.am
@@ -33,7 +33,8 @@
score.c send.c sendlib.c signal.c sort.c \
status.c system.c thread.c charset.c history.c lib.c \
muttlib.c editmsg.c mbyte.c \
- url.c ascii.c crypt-mod.c crypt-mod.h safe_asprintf.c
+ url.c ascii.c crypt-mod.c crypt-mod.h safe_asprintf.c \
+ mutt_socket_compress.c
nodist_mutt_SOURCES = $(BUILT_SOURCES)
@@ -73,7 +74,7 @@
README.SSL smime.h group.h \
muttbug pgppacket.h depcomp ascii.h BEWARE PATCHES patchlist.sh \
ChangeLog mkchangelog.sh mutt_idna.h \
- snprintf.c regex.c crypt-gpgme.h hcachever.sh.in
+ snprintf.c regex.c crypt-gpgme.h hcachever.sh.in mutt_socket_compress.h
EXTRA_SCRIPTS = smime_keys
diff --git a/configure.ac b/configure.ac
--- a/configure.ac
+++ b/configure.ac
@@ -652,6 +652,8 @@
dnl -- end imap dependencies --
+AC_CHECK_LIB(z, deflate)
+
AC_ARG_WITH(ssl, AC_HELP_STRING([--with-ssl@<:@=PFX@:>@], [Enable TLS support
using OpenSSL]),
[ if test "$with_ssl" != "no"
then
@@ -666,7 +668,6 @@
saved_LIBS="$LIBS"
crypto_libs=""
- AC_CHECK_LIB(z, deflate, [crypto_libs=-lz])
AC_CHECK_LIB(crypto, X509_new,
[crypto_libs="-lcrypto $crypto_libs"],, [$crypto_libs])
AC_CHECK_LIB(ssl, SSL_new,,
diff --git a/imap/command.c b/imap/command.c
--- a/imap/command.c
+++ b/imap/command.c
@@ -66,6 +66,7 @@
"LOGINDISABLED",
"IDLE",
"SASL-IR",
+ "COMPRESS=DEFLATE",
NULL
};
diff --git a/imap/imap.c b/imap/imap.c
--- a/imap/imap.c
+++ b/imap/imap.c
@@ -39,6 +39,7 @@
#if USE_HCACHE
#include "hcache.h"
#endif
+#include "mutt_socket_compress.h"
#include <unistd.h>
#include <ctype.h>
@@ -316,6 +317,33 @@
return 0;
}
+static void enable_compression (IMAP_DATA *idata)
+{
+#ifdef HAVE_LIBZ
+ if (option (OPTIMAPCOMPRESS) && mutt_bit_isset (idata->capabilities,
COMPRESS_DEFLATE))
+ {
+ int rc = IMAP_CMD_CONTINUE;
+ char *s;
+
+ dprint (1, (debugfile, "requesting IMAP COMPRESS=DEFLATE\n"));
+ imap_cmd_start (idata, "COMPRESS DEFLATE");
+
+ do
+ {
+ if ((rc = imap_cmd_step (idata) == IMAP_CMD_CONTINUE))
+ continue;
+ if (rc == IMAP_CMD_OK)
+ {
+ dprint (1, (debugfile, "successfully negotiated COMPRESS=DEFLATE\n"));
+ mutt_socket_compress (idata->conn);
+ }
+ else
+ dprint (1, (debugfile, "unable to negotiate COMPRESS=DEFLATE\n"));
+ } while (rc == IMAP_CMD_CONTINUE);
+ }
+#endif
+}
+
/* imap_conn_find: Find an open IMAP connection matching account, or open
* a new one if none can be found. */
IMAP_DATA* imap_conn_find (const ACCOUNT* account, int flags)
@@ -388,13 +416,17 @@
{
/* capabilities may have changed */
imap_exec (idata, "CAPABILITY", IMAP_CMD_QUEUE);
+
/* get root delimiter, '/' as default */
idata->delim = '/';
imap_exec (idata, "LIST \"\" \"\"", IMAP_CMD_QUEUE);
if (option (OPTIMAPCHECKSUBSCRIBED))
imap_exec (idata, "LSUB \"\" \"*\"", IMAP_CMD_QUEUE);
+
/* we may need the root delimiter before we open a mailbox */
imap_exec (idata, NULL, IMAP_CMD_FAIL_OK);
+
+ enable_compression (idata);
}
return idata;
diff --git a/imap/imap_private.h b/imap/imap_private.h
--- a/imap/imap_private.h
+++ b/imap/imap_private.h
@@ -114,6 +114,7 @@
LOGINDISABLED, /* LOGINDISABLED */
IDLE, /* RFC 2177: IDLE */
SASL_IR, /* SASL initial response draft */
+ COMPRESS_DEFLATE, /* RFC 4978: IMAP COMPRESS extension */
CAPMAX
};
@@ -206,6 +207,7 @@
char *mailbox;
unsigned short check_status;
unsigned char reopen;
+ unsigned char compress; /* IMAP compression */
unsigned int newMailCount;
IMAP_CACHE cache[IMAP_CACHE_LEN];
unsigned int uid_validity;
diff --git a/init.h b/init.h
--- a/init.h
+++ b/init.h
@@ -1083,6 +1083,18 @@
** it polls for new mail just as if you had issued individual ``$mailboxes''
** commands.
*/
+#ifdef HAVE_LIBZ
+ { "imap_compress", DT_BOOL, R_NONE, OPTIMAPCOMPRESS, 0 },
+ /*
+ ** .pp
+ ** When \fIset\fP, Mutt will enable compression when communicating with
+ ** IMAP servers that support it.
+ ** .pp
+ ** If \fI$$ssl_compress\fP is also set, Mutt will try to use TLS-layer
+ ** compression, and fall back to IMAP compression if not supported on
+ ** the server.
+ */
+#endif /* HAVE_LIBZ */
{ "imap_delim_chars", DT_STR, R_NONE, UL &ImapDelimChars, UL
"/." },
/*
** .pp
diff --git a/main.c b/main.c
--- a/main.c
+++ b/main.c
@@ -310,6 +310,12 @@
"-USE_GNU_REGEX "
#endif
+#ifdef HAVE_LIBZ
+ "+HAVE_LIBZ "
+#else
+ "-HAVE_LIBZ "
+#endif
+
"\n"
#ifdef HAVE_COLOR
diff --git a/mutt.h b/mutt.h
--- a/mutt.h
+++ b/mutt.h
@@ -363,6 +363,7 @@
OPTIGNORELISTREPLYTO,
#ifdef USE_IMAP
OPTIMAPCHECKSUBSCRIBED,
+ OPTIMAPCOMPRESS,
OPTIMAPIDLE,
OPTIMAPLSUB,
OPTIMAPPASSIVE,
diff --git a/mutt_socket_compress.c b/mutt_socket_compress.c
new file mode 100644
--- /dev/null
+++ b/mutt_socket_compress.c
@@ -0,0 +1,277 @@
+/*
+ * Copyright (C) 2010 Michael R. Elkins <me@xxxxxxxx>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
02110-1301, USA.
+ */
+
+/*
+ * This file contains support for compressing/decompressing data streams using
+ * the mutt_socket interface. It is intended to support the IMAP
+ * COMPRESS=DEFLATE extension.
+ */
+
+/*
+ * FIXME: the conn_*() functions really should be put into their own data
+ * structure rather than living directly in CONNECTION, so that the push/pop
+ * functions are not necessary.
+ */
+
+#if HAVE_CONFIG_H
+# include "config.h"
+#endif
+
+#if defined(HAVE_LIBZ) && defined(USE_IMAP)
+
+#include <zlib.h>
+
+#include "mutt.h"
+#include "mutt_socket.h"
+
+typedef struct _compdata
+{
+ z_stream in;
+ z_stream out;
+
+ char *inb;
+ size_t inbsize;
+
+ char *outb;
+ size_t outbsize;
+
+ void *sockdata; /* data for underlying driver */
+
+ int (*conn_read) (struct _connection* conn, char* buf, size_t len);
+ int (*conn_write) (struct _connection *conn, const char *buf, size_t count);
+ int (*conn_open) (struct _connection *conn);
+ int (*conn_close) (struct _connection *conn);
+ int (*conn_poll) (struct _connection *conn);
+
+} compdata;
+
+static void socket_driver_push (CONNECTION *conn, compdata *cdata);
+static void socket_driver_pop (CONNECTION *conn);
+
+static int socket_compress_poll (CONNECTION* conn)
+{
+ int r;
+
+ compdata* cdata = conn->sockdata;
+
+ /*
+ * When decompressing, the buffer the caller used in mutt_socket_read() might
+ * have been too short. In this case, there will be left over data to be
+ * read.
+ */
+ if (cdata->in.avail_in > 0)
+ return 1;
+
+ socket_driver_pop (conn);
+
+ r = mutt_socket_poll (conn);
+
+ socket_driver_push (conn, cdata);
+
+ return r;
+}
+
+static int socket_compress_read (CONNECTION* conn, char* buf, size_t len)
+{
+ int r;
+ int zr;
+ int flush = Z_SYNC_FLUSH;
+
+ compdata* cdata = conn->sockdata;
+ socket_driver_pop (conn);
+
+ /* check for data left over from last call */
+ if (cdata->in.avail_in == 0)
+ {
+ if (cdata->inbsize < len + 6)
+ {
+ cdata->inbsize = len + 6;
+ safe_realloc (&cdata->inb, cdata->inbsize);
+ }
+
+ /* refill */
+ r = mutt_socket_read (conn, cdata->inb, cdata->inbsize);
+ if (r > 0)
+ {
+ cdata->in.next_in = cdata->inb;
+ cdata->in.avail_in = r;
+ }
+ else
+ goto error_out_0;
+ }
+ /* else there was data left over */
+
+ cdata->in.next_out = buf;
+ cdata->in.avail_out = len;
+
+ zr = inflate (&cdata->in, flush);
+ if (zr == Z_OK)
+ {
+ r = len - cdata->in.avail_out;
+
+ if (cdata->in.avail_out > 0)
+ {
+ /* zlib manual doesn't explicitly say what happens when all data was
consumed */
+ cdata->in.avail_in = 0;
+ }
+
+ if (r == 0)
+ {
+ dprint(1, (debugfile, "oops, inflate returned 0 bytes!\n"));
+ }
+ }
+ else
+ {
+ dprint (1, (debugfile, "inflate() returned %d\n", zr));
+ r = -1;
+ }
+
+error_out_0:
+ socket_driver_push (conn, cdata);
+
+ return r;
+}
+
+static int socket_compress_write (CONNECTION *conn, const char *buf, size_t
len)
+{
+ int zr;
+ int r = -1;
+ int flush = Z_SYNC_FLUSH;
+ compdata* cdata = conn->sockdata;
+
+ socket_driver_pop (conn);
+
+ if (cdata->outbsize < len + 6)
+ {
+ cdata->outbsize = len + 6;
+ safe_realloc (&cdata->outb, cdata->outbsize);
+ }
+
+ cdata->out.next_in = buf;
+ cdata->out.avail_in = len;
+
+ do {
+ cdata->out.next_out = cdata->outb;
+ cdata->out.avail_out = cdata->outbsize;
+
+ zr = deflate (&cdata->out, flush);
+ if (zr == Z_OK && cdata->out.avail_out == 0)
+ {
+ /* more space required */
+ cdata->outbsize *= 2;
+ safe_realloc (&cdata->outb, cdata->outbsize);
+ }
+ else
+ break;
+ } while (1);
+
+ if (zr == Z_OK)
+ {
+ /* guarantees all data will be written */
+ if (mutt_socket_write_n (conn, cdata->outb, cdata->outbsize -
cdata->out.avail_out) > 0)
+ r = len; /* return what the user requested, not the compressed size sent
*/
+ }
+ else
+ {
+ dprint (1, (debugfile, "deflate() returned %d\n", zr));
+ }
+
+ socket_driver_push (conn, cdata);
+
+ return r;
+}
+
+static int socket_compress_close (CONNECTION *conn)
+{
+ int r;
+
+ compdata* cdata = conn->sockdata;
+ socket_driver_pop (conn);
+
+ r = mutt_socket_close (conn);
+
+ dprint (1, (debugfile, "compression stats: sent raw=%lu/real=%lu (%d%%)
received raw=%lu/real=%lu (%d%%)\n",
+ cdata->out.total_in, cdata->out.total_out,
+ /* upstream compresses, so out will be smaller than in (hopefully!) */
+ cdata->out.total_in > 0 ? (int) (100.0 * cdata->out.total_out /
(float)cdata->out.total_in) : 0,
+ cdata->in.total_out, cdata->in.total_in,
+ cdata->in.total_out > 0 ? (int) (100.0 * cdata->in.total_in / (float)
cdata->in.total_out) : 0));
+
+ deflateEnd (&cdata->out);
+ inflateEnd (&cdata->in);
+
+ FREE (&cdata);
+
+ return r;
+}
+
+/*
+ * Compresses and decompresses all data sent/received on this connection.
+ * It *must* be called after the connection is opened, as it has no defined
+ * open() method!
+ */
+void mutt_socket_compress (CONNECTION *conn)
+{
+ compdata *cdata = safe_calloc (1, sizeof (compdata));
+ socket_driver_push (conn, cdata);
+
+ /* the magic constant -15 is specified in RFC 4978, so that zlib will use raw
+ * DEFLATE with no framing.
+ */
+ deflateInit2 (&cdata->out, Z_DEFAULT_COMPRESSION, Z_DEFLATED, -15, 8,
Z_DEFAULT_STRATEGY);
+ inflateInit2 (&cdata->in, -15);
+}
+
+/*
+ * Replace the current socket driver with the compression driver. The current
+ * driver is saved and restored when the compression driver needs to call the
+ * lower level driver.
+ */
+static void socket_driver_push (CONNECTION *conn, compdata *cdata)
+{
+ cdata->sockdata = conn->sockdata;
+ conn->sockdata = cdata;
+
+ cdata->conn_read = conn->conn_read;
+ cdata->conn_write = conn->conn_write;
+ cdata->conn_close = conn->conn_close;
+ cdata->conn_poll = conn->conn_poll;
+
+ conn->conn_read = socket_compress_read;
+ conn->conn_write = socket_compress_write;
+ conn->conn_close = socket_compress_close;
+ conn->conn_poll = socket_compress_poll;
+}
+
+/*
+ * Removes the current installed compression driver and restores the saved
+ * lower level socket driver. used prior to making calls into the lower level
+ * driver.
+ */
+static void socket_driver_pop (CONNECTION *conn)
+{
+ compdata* cdata = conn->sockdata;
+
+ conn->sockdata = cdata->sockdata;
+ conn->conn_read = cdata->conn_read;
+ conn->conn_write = cdata->conn_write;
+ conn->conn_close = cdata->conn_close;
+ conn->conn_poll = cdata->conn_poll;
+}
+
+#endif /* HAVE_LIBZ */
diff --git a/mutt_socket_compress.h b/mutt_socket_compress.h
new file mode 100644
--- /dev/null
+++ b/mutt_socket_compress.h
@@ -0,0 +1,26 @@
+/*
+ * Copyright (C) 2010 Michael R. Elkins <me@xxxxxxxx>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
02110-1301, USA.
+ */
+
+#ifndef mutt_socket_compress_h
+#define mutt_socket_compress_h
+
+#if defined(HAVE_LIBZ) && defined(USE_IMAP)
+void mutt_socket_compress (CONNECTION *conn);
+#endif
+
+#endif