diff -Nru a/PATCHES b/PATCHES --- a/PATCHES +++ b/PATCHES @@ -0,0 +1 @@ +patch-1.5.6.tg.hcache.11 diff -Nru a/configure.in b/configure.in --- a/configure.in 2004-08-18 09:43:09 +02:00 +++ b/configure.in 2004-10-14 07:21:52 +02:00 @@ -692,11 +692,6 @@ AC_CHECK_FUNCS(idna_to_ascii_lz idna_to_ascii_from_locale) fi -if test "$need_md5" = "yes" -then - MUTT_LIB_OBJECTS="$MUTT_LIB_OBJECTS md5c.o" -fi - dnl -- end socket -- AC_ARG_ENABLE(debug, [ --enable-debug Enable debugging support], @@ -767,6 +762,96 @@ NOTE: this requires significant more memory when defined.]) fi]) + +dnl -- start cache -- +AC_ARG_ENABLE(hcache, [ --enable-hcache Enable header caching], +[if test x$enableval = xyes; then + AC_DEFINE(USE_HCACHE, 1, [Enable header caching]) + MUTT_LIB_OBJECTS="$MUTT_LIB_OBJECTS hcache.o" + + OLDCPPFLAGS="$CPPFLAGS" + OLDLIBS="$LIBS" + + need_md5="yes" + ac_prefer_gdbm=yes + AC_ARG_WITH(gdbm, [ --without-gdbm Get rid of gdbm even if it is available ], + ac_prefer_gdbm=$withval) + if test x$ac_prefer_gdbm != xno; then + CPPFLAGS="$OLDCPPFLAGS" + LIBS="$OLDLIBS -lgdbm"; + AC_CACHE_CHECK(for gdbm_open, ac_cv_gdbmopen,[ + ac_cv_gdbmopen=no + AC_TRY_LINK([#include ],[gdbm_open(0,0,0,0,0);],[ac_cv_gdbmopen=yes]) + ]) + fi + + ac_bdb_prefix=yes + AC_ARG_WITH(bdb, [ --with-bdb[=DIR] Use BerkeleyDB4 if gdbm is not available ], + ac_bdb_prefix=$withval) + if test x$ac_bdb_prefix != xno -a x$ac_cv_gdbmopen != xyes; then + test x$ac_bdb_prefix = xyes && ac_bdb_prefix="$mutt_cv_prefix /opt/csw/bdb4 /opt /usr/local /usr" + for d in $ac_bdb_prefix; do + bdbpfx="$bdbpfx $d" + for v in BerkeleyDB.4.3 BerkeleyDB.4.2 BerkeleyDB.4.1; do + bdbpfx="$bdbpfx $d/$v" + done + done + BDB_VERSIONS="db-4.2 db4.2 db42 ''" + AC_MSG_CHECKING([for BerkeleyDB > 4.0]) + for d in $bdbpfx; do + BDB_INCLUDE_DIR="" + BDB_LIB_DIR="" + for v in / $BDB_VERSIONS; do + if test -r "$d/include/$v/db.h"; then + BDB_INCLUDE_DIR="$d/include/$v" + if test -d "$d/lib/$v"; then + BDB_LIB_DIR="$d/lib/$v" + for l in `echo $BDB_VERSIONS`; do + CPPFLAGS="-I$BDB_INCLUDE_DIR $OLDCPPFLAGS" + LIBS="$OLDLIBS -L$BDB_LIB_DIR -l$l" + AC_TRY_LINK([ + #include + #include + ],[ + DB *db = NULL; + db->open(db,NULL,NULL,NULL,0,0,0); + ],[ + ac_cv_dbcreate=yes + BDB_LIB="$l" + break + ]) + done + test x$ac_dbcreate = xyes && break + fi + fi + done + test x$BDB_LIB != x && break + done + if test x$ac_cv_dbcreate = xyes; then + AC_MSG_RESULT(yes) + else + AC_MSG_RESULT(no) + fi + fi + + if test x$ac_cv_gdbmopen = xyes; then + CPPFLAGS="$OLDCPPFLAGS" + LIBS="$OLDLIBS -lgdbm"; + AC_DEFINE(HAVE_GDBM, 1, [GDBM Support]) + elif test x$ac_cv_dbcreate = xyes; then + CPPFLAGS="-I$BDB_INCLUDE_DIR $OLDCPPFLAGS" + LIBS="$OLDLIBS -L$BDB_LIB_DIR -l$BDB_LIB" + AC_DEFINE(HAVE_DB4, 1, [Sleepycat DB4 Support]) + else + AC_MSG_ERROR(You need Sleepycat DB4 or GDBM for --enable-hcache) + fi +fi]) +dnl -- end cache -- + +if test "$need_md5" = "yes" +then + MUTT_LIB_OBJECTS="$MUTT_LIB_OBJECTS md5c.o" +fi AC_SUBST(MUTTLIBS) AC_SUBST(MUTT_LIB_OBJECTS) diff -Nru a/globals.h b/globals.h --- a/globals.h 2004-08-18 09:43:11 +02:00 +++ b/globals.h 2004-08-26 22:32:53 +02:00 @@ -63,6 +63,10 @@ WHERE char *Locale; WHERE char *MailcapPath; WHERE char *Maildir; +#if USE_HCACHE +WHERE char *HeaderCache; +WHERE char *HeaderCachePageSize; +#endif WHERE char *MhFlagged; WHERE char *MhReplied; WHERE char *MhUnseen; diff -Nru a/hcache.c b/hcache.c --- /dev/null Wed Dec 31 16:00:00 196900 +++ b/hcache.c 2004-11-09 00:09:22 +01:00 @@ -0,0 +1,880 @@ +/* + * Copyright (C) 2004 Thomas Glanzmann + * Copyright (C) 2004 Tobias Werth + * Copyright (C) 2004 Brian Fundakowski Feldman + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111, USA. + */ + +#if HAVE_CONFIG_H +#include "config.h" +#endif /* HAVE_CONFIG_H */ + +#if HAVE_GDBM +#include +#elif HAVE_DB4 +#include +#endif + +#include +#include +#include "mutt.h" +#ifdef USE_IMAP +#include "message.h" +#endif +#include "mime.h" +#include "mx.h" +#include "lib.h" +#include "md5.h" + +#if HAVE_GDBM +static struct +header_cache +{ + GDBM_FILE db; + char *folder; + unsigned int crc; +} HEADER_CACHE; +#elif HAVE_DB4 +static struct +header_cache +{ + DB_ENV *env; + DB *db; + unsigned int crc; + int fd; + char lockfile[_POSIX_PATH_MAX]; +} HEADER_CACHE; +#endif + +typedef union +{ + struct timeval timeval; + unsigned long long uid_validity; +} validate; + +static void * +lazy_malloc(size_t siz) +{ + if (0 < siz && siz < 4096) { + siz = 4096; + } + + return safe_malloc(siz); +} + +static void +lazy_realloc(void *ptr, size_t siz) +{ + void **p = (void **)ptr; + + if ( p != NULL + && 0 < siz + && siz < 4096) { + return; + } + + safe_realloc(ptr, siz); +} + +static unsigned char * +dump_int(unsigned int i, unsigned char *d, int *off) +{ + lazy_realloc(&d, *off + sizeof(int)); + memcpy(d + *off, &i, sizeof(int)); + (*off) += sizeof(int); + + return d; +} + +static void +restore_int(unsigned int *i, const unsigned char *d, int *off) +{ + memcpy(i, d + *off, sizeof(int)); + (*off) += sizeof(int); +} + +static unsigned char * +dump_char(char *c, unsigned char *d, int *off) +{ + unsigned int size; + + if (c == NULL) { + size = 0; + d = dump_int(size, d, off); + return d; + } + + size = mutt_strlen(c) + 1; + d = dump_int(size, d, off); + lazy_realloc(&d, *off + size); + memcpy(d + *off, c, size); + *off += size; + + return d; +} + +static unsigned char * +dump_char_size(char *c, unsigned char *d, int *off, ssize_t size) +{ + if (c == NULL) { + size = 0; + d = dump_int(size, d, off); + return d; + } + + d = dump_int(size, d, off); + lazy_realloc(&d, *off + size); + memcpy(d + *off, c, size); + *off += size; + + return d; +} + +static void +restore_char(char **c, const unsigned char *d, int *off) +{ + unsigned int size; + restore_int(&size, d, off); + + if (size == 0) { + *c = NULL; + return; + } + + *c = safe_malloc(size); + memcpy(*c, d + *off, size); + *off += size; +} + +static unsigned char * +dump_address(ADDRESS *a, unsigned char *d, int *off) +{ + unsigned int counter = 0; + unsigned int start_off = *off; + + d = dump_int(0xdeadbeef, d, off); + + while (a) { +#ifdef EXACT_ADDRESS + d = dump_char(a->val, d, off); +#endif + d = dump_char(a->personal, d, off); + d = dump_char(a->mailbox, d, off); + d = dump_int(a->group, d, off); + a = a->next; + counter++; + } + + memcpy(d + start_off, &counter, sizeof(int)); + + return d; +} + +static void +restore_address(ADDRESS **a, const unsigned char *d, int *off) +{ + unsigned int counter; + + restore_int(&counter, d, off); + + while (counter) { + *a = safe_malloc(sizeof(ADDRESS)); +#ifdef EXACT_ADDRESS + restore_char(&(*a)->val, d, off); +#endif + restore_char(&(*a)->personal, d, off); + restore_char(&(*a)->mailbox, d, off); + restore_int((unsigned int *)&(*a)->group, d, off); + a = &(*a)->next; + counter--; + } + + *a = NULL; +} + +static unsigned char * +dump_list(LIST *l, unsigned char *d, int *off) +{ + unsigned int counter = 0; + unsigned int start_off = *off; + + d = dump_int(0xdeadbeef, d, off); + + while (l) { + d = dump_char(l->data, d, off); + l = l->next; + counter++; + } + + memcpy(d + start_off, &counter, sizeof(int)); + + return d; +} + +static void +restore_list(LIST **l, const unsigned char *d, int *off) +{ + unsigned int counter; + + restore_int(&counter, d, off); + + while (counter) { + *l = safe_malloc(sizeof(LIST)); + restore_char(&(*l)->data, d, off); + l = &(*l)->next; + counter--; + } + + *l = NULL; +} + +static unsigned char * +dump_buffer(BUFFER *b, unsigned char *d, int *off) +{ + if (! b) { + d = dump_int(0, d, off); + return d; + } else { + d = dump_int(1, d, off); + } + + d = dump_char_size(b->data, d, off, b->dsize + 1); + d = dump_int(b->dptr - b->data, d, off); + d = dump_int(b->dsize, d, off); + d = dump_int(b->destroy, d, off); + + return d; +} + +static void +restore_buffer(BUFFER **b, const unsigned char *d, int *off) +{ + unsigned int used; + unsigned int offset; + restore_int(&used, d, off); + if (! used) { + return; + } + + *b = safe_malloc(sizeof(BUFFER)); + + restore_char(& (*b)->data, d, off); + restore_int(& offset, d, off); + (*b)->dptr = (*b)->data + offset; + restore_int(& (*b)->dsize, d, off); + restore_int((unsigned int *) & (*b)->destroy, d, off); +} + +static unsigned char * +dump_parameter(PARAMETER *p, unsigned char *d, int *off) +{ + unsigned int counter = 0; + unsigned int start_off = *off; + + d = dump_int(0xdeadbeef, d, off); + + while (p) { + d = dump_char(p->attribute, d, off); + d = dump_char(p->value, d, off); + p = p->next; + counter++; + } + + memcpy(d + start_off, &counter, sizeof(int)); + + return d; +} + +static void +restore_parameter(PARAMETER **p, const unsigned char *d, int *off) +{ + unsigned int counter; + + restore_int(&counter, d, off); + + while (counter) { + *p = safe_malloc(sizeof(PARAMETER)); + restore_char(&(*p)->attribute, d, off); + restore_char(&(*p)->value, d, off); + p = &(*p)->next; + counter--; + } + + *p = NULL; +} + +static unsigned char * +dump_body(BODY *c, unsigned char *d, int *off) +{ + lazy_realloc(&d, *off + sizeof(BODY)); + memcpy(d + *off, c, sizeof(BODY)); + *off += sizeof(BODY); + + d = dump_char(c->xtype, d, off); + d = dump_char(c->subtype, d, off); + + d = dump_parameter(c->parameter, d, off); + + d = dump_char(c->description, d, off); + d = dump_char(c->form_name, d, off); + d = dump_char(c->filename, d, off); + d = dump_char(c->d_filename, d, off); + + return d; +} + +static void +restore_body(BODY *c, const unsigned char *d, int *off) +{ + memcpy(c, d + *off, sizeof(BODY)); + *off += sizeof(BODY); + + restore_char(& c->xtype, d, off); + restore_char(& c->subtype, d, off); + + restore_parameter(& c->parameter, d, off); + + restore_char(& c->description, d, off); + restore_char(& c->form_name, d, off); + restore_char(& c->filename, d, off); + restore_char(& c->d_filename, d, off); +} + +static unsigned char * +dump_envelope(ENVELOPE *e, unsigned char *d, int *off) +{ + d = dump_address(e->return_path, d, off); + d = dump_address(e->from, d, off); + d = dump_address(e->to, d, off); + d = dump_address(e->cc, d, off); + d = dump_address(e->bcc, d, off); + d = dump_address(e->sender, d, off); + d = dump_address(e->reply_to, d, off); + d = dump_address(e->mail_followup_to, d, off); + + d = dump_char(e->subject, d, off); + if (e->real_subj) { + d = dump_int(e->real_subj - e->subject, d, off); + } else { + d = dump_int(-1, d, off); + } + d = dump_char(e->message_id, d, off); + d = dump_char(e->supersedes, d, off); + d = dump_char(e->date, d, off); + d = dump_char(e->x_label, d, off); + + d = dump_list(e->references, d, off); + d = dump_list(e->in_reply_to, d, off); + d = dump_list(e->userhdrs, d, off); + + return d; +} + +static void +restore_envelope(ENVELOPE *e, const unsigned char *d, int *off) +{ + int real_subj_off; + + restore_address(& e->return_path, d, off); + restore_address(& e->from, d, off); + restore_address(& e->to, d, off); + restore_address(& e->cc, d, off); + restore_address(& e->bcc, d, off); + restore_address(& e->sender, d, off); + restore_address(& e->reply_to, d, off); + restore_address(& e->mail_followup_to, d, off); + + restore_char(& e->subject, d, off); + restore_int((unsigned int *) (& real_subj_off), d, off); + if (0 <= real_subj_off) { + e->real_subj = e->subject + real_subj_off; + } else { + e->real_subj = NULL; + } + restore_char(& e->message_id, d, off); + restore_char(& e->supersedes, d, off); + restore_char(& e->date, d, off); + restore_char(& e->x_label, d, off); + + restore_list(& e->references, d, off); + restore_list(& e->in_reply_to, d, off); + restore_list(& e->userhdrs, d, off); +} + +static +unsigned int crc32(unsigned int crc, unsigned char const *p, size_t len) +{ + int i; + while (len--) { + crc ^= *p++; + for (i = 0; i < 8; i++) + crc = (crc >> 1) ^ ((crc & 1) ? 0xedb88320 : 0); + } + return crc; +} + +static int +generate_crc32() +{ + int crc = 0; + + crc = crc32(crc, (unsigned char const *) "sithglan@stud.uni-erlangen.de[sithglan]|hcache.c|20041108231548|29613", mutt_strlen("sithglan@stud.uni-erlangen.de[sithglan]|hcache.c|20041108231548|29613")); + +#if HAVE_LANGINFO_CODESET + crc = crc32(crc, (unsigned char const *) Charset, mutt_strlen(Charset)); + crc = crc32(crc, (unsigned char const *) "HAVE_LANGINFO_CODESET", mutt_strlen("HAVE_LANGINFO_CODESET")); +#endif + +#if EXACT_ADDRESS + crc = crc32(crc, (unsigned char const *) "EXACT_ADDRESS", mutt_strlen("EXACT_ADDRESS")); +#endif + +#ifdef USE_POP + crc = crc32(crc, (unsigned char const *) "USE_POP", mutt_strlen("USE_POP")); +#endif + +#ifdef MIXMASTER + crc = crc32(crc, (unsigned char const *) "MIXMASTER", mutt_strlen("MIXMASTER")); +#endif + +#ifdef USE_IMAP + crc = crc32(crc, (unsigned char const *) "USE_IMAP", mutt_strlen("USE_IMAP")); +#endif + return crc; +} + +static int +crc32_matches(const char *d, unsigned int crc) +{ + int off = sizeof(validate); + unsigned int mycrc = 0; + + if (! d) { + return 0; + } + + restore_int(&mycrc, (unsigned char *) d, &off); + + return (crc == mycrc); +} + +/* Append md5sumed folder to path if path is a directory. */ +static const char * +mutt_hcache_per_folder(const char *path, const char *folder) +{ + static char mutt_hcache_per_folder_path[_POSIX_PATH_MAX]; + struct stat path_stat; + MD5_CTX md5; + unsigned char md5sum[16]; + int ret; + + ret = stat(path, &path_stat); + if (ret < 0) { + return path; + } + + if (! S_ISDIR(path_stat.st_mode)) { + return path; + } + + MD5Init(&md5); + MD5Update(&md5, (unsigned char *) folder, strlen(folder)); + MD5Final(md5sum, &md5); + + ret = snprintf(mutt_hcache_per_folder_path, _POSIX_PATH_MAX, + "%s/%02x%02x%02x%02x%02x%02x%02x%02x" + "%02x%02x%02x%02x%02x%02x%02x%02x", + path, md5sum[0], md5sum[1], md5sum[2], md5sum[3], + md5sum[4], md5sum[5], md5sum[6], md5sum[7], md5sum[8], + md5sum[9], md5sum[10], md5sum[11], md5sum[12], + md5sum[13], md5sum[14], md5sum[15]); + + if (ret <= 0) { + return path; + } + + return mutt_hcache_per_folder_path; +} + +/* This function transforms a header into a char so that it is useable by + * db_store */ +static void * +mutt_hcache_dump(void *_db, HEADER *h, int *off, unsigned long long uid_validity) +{ + struct header_cache *db = _db; + unsigned char *d = NULL; + *off = 0; + + d = lazy_malloc(sizeof(validate)); + + if (uid_validity) { + memcpy(d, &uid_validity, sizeof(long long)); + } else { + struct timeval now; + gettimeofday(&now, NULL); + memcpy(d, &now, sizeof(struct timeval)); + } + *off += sizeof(validate); + + d = dump_int(db->crc, d, off); + + lazy_realloc(&d, *off + sizeof(HEADER)); + memcpy(d + *off, h, sizeof(HEADER)); + *off += sizeof(HEADER); + + d = dump_envelope(h->env, d, off); + d = dump_body(h->content, d, off); + d = dump_char(h->maildir_flags, d, off); + + return d; +} + +HEADER * +mutt_hcache_restore(const unsigned char *d, HEADER **oh) +{ + int off = 0; + HEADER *h = mutt_new_header(); + + /* skip validate */ + off += sizeof(validate); + + /* skip crc */ + off += sizeof(unsigned int); + + memcpy(h, d + off, sizeof(HEADER)); + off += sizeof(HEADER); + + h->env = mutt_new_envelope(); + restore_envelope(h->env, d, &off); + + h->content = mutt_new_body(); + restore_body(h->content, d, &off); + + restore_char(&h->maildir_flags, d, &off); + + /* this is needed for maildir style mailboxes */ + if (oh) { + h->old = (*oh)->old; + h->path = safe_strdup((*oh)->path); + mutt_free_header (oh); + } + + return h; +} + +#if HAVE_GDBM + +void * +mutt_hcache_open(const char *path, const char *folder) +{ + struct header_cache *h = safe_calloc(1, sizeof(HEADER_CACHE)); + int pagesize = atoi(HeaderCachePageSize) ? atoi(HeaderCachePageSize) : 16384; + h->db = NULL; + h->folder = safe_strdup (folder); + h->crc = generate_crc32(); + + if (! path || path[0] == '\0') { + FREE(& h->folder); + FREE(& h); + return NULL; + } + + path = mutt_hcache_per_folder(path, folder); + + h->db = gdbm_open((char *) path, pagesize, GDBM_WRCREAT, 00600, NULL); + if (h->db) { + return h; + } + + /* if rw failed try ro */ + h->db = gdbm_open((char *) path, pagesize, GDBM_READER, 00600, NULL); + if(h->db) { + return h; + } else { + FREE(& h->folder); + FREE(& h); + + return NULL; + } +} + +void +mutt_hcache_close(void *db) +{ + struct header_cache *h = db; + + if (! h) { + return; + } + + gdbm_close(h->db); + FREE(& h->folder); + FREE(& h); +} + +void * +mutt_hcache_fetch(void *db, const char *filename, size_t (*keylen)(const char *fn)) +{ + struct header_cache *h = db; + datum key; + datum data; + char path[_POSIX_PATH_MAX]; + + if (! h) { + return NULL; + } + + strncpy(path, h->folder, sizeof(path)); + strncat(path, filename, sizeof(path) - mutt_strlen(path)); + + key.dptr = path; + key.dsize = keylen(path); + + data = gdbm_fetch(h->db, key); + + if (! crc32_matches(data.dptr, h->crc)) { + free(data.dptr); + return NULL; + } + + return data.dptr; +} + +int +mutt_hcache_store(void *db, const char *filename, HEADER *header, unsigned long long uid_validity, size_t (*keylen)(const char *fn)) +{ + struct header_cache *h = db; + datum key; + datum data; + char path[_POSIX_PATH_MAX]; + int ret; + + if (! h) { + return -1; + } + + strncpy(path, h->folder, sizeof(path)); + strncat(path, filename, sizeof(path) - mutt_strlen(path)); + + key.dptr = path; + key.dsize = keylen(path); + + data.dptr = mutt_hcache_dump(db, header, &data.dsize, uid_validity); + + ret = gdbm_store(h->db, key, data, GDBM_REPLACE); + + FREE(& data.dptr); + + return ret; +} + +int +mutt_hcache_delete(void *db, const char *filename, size_t (*keylen)(const char *fn)) +{ + datum key; + struct header_cache *h = db; + char path[_POSIX_PATH_MAX]; + + if (! h) { + return -1; + } + + strncpy(path, h->folder, sizeof(path)); + strncat(path, filename, sizeof(path) - mutt_strlen(path)); + + key.dptr = path; + key.dsize = keylen(path); + + return gdbm_delete(h->db, key); +} +#elif HAVE_DB4 + +static void +mutt_hcache_dbt_init(DBT *dbt, void *data, size_t len) +{ + dbt->data = data; + dbt->size = dbt->ulen = len; + dbt->dlen = dbt->doff = 0; + dbt->flags = DB_DBT_USERMEM; +} + +static void +mutt_hcache_dbt_empty_init(DBT *dbt) +{ + dbt->data = NULL; + dbt->size = dbt->ulen = dbt->dlen = dbt->doff = 0; + dbt->flags = 0; +} + +void * +mutt_hcache_open(const char *path, const char *folder) +{ + struct stat sb; + u_int32_t createflags = DB_CREATE; + int ret; + struct header_cache *h = calloc(1, sizeof(HEADER_CACHE)); + int pagesize = atoi(HeaderCachePageSize); + + + h->crc = generate_crc32(); + + if (! path || path[0] == '\0') { + FREE(& h); + return NULL; + } + + path = mutt_hcache_per_folder(path, folder); + + snprintf (h->lockfile, _POSIX_PATH_MAX, "%s-lock-hack", path); + + h->fd = open(h->lockfile, O_WRONLY | O_CREAT, S_IRUSR | S_IWUSR); + if (h->fd < 0) { + FREE (&h); + return NULL; + } + + if (mx_lock_file(h->lockfile, h->fd, 1, 0, 5)) { + close(h->fd); + FREE (&h); + return NULL; + } + + ret = db_env_create(&h->env, 0); + if (ret) { + mx_unlock_file(h->lockfile, h->fd, 0); + close(h->fd); + FREE(& h); + return NULL; + } + + ret = h->env->open(h->env, NULL, DB_INIT_MPOOL | DB_CREATE | DB_PRIVATE, 0600); + if (! ret) { + ret = db_create(&h->db, h->env, 0); + if (ret) { + h->env->close(h->env, 0); + mx_unlock_file(h->lockfile, h->fd, 0); + close(h->fd); + FREE(& h); + return NULL; + } + } + + if (stat(path, &sb) != 0 && errno == ENOENT) { + createflags |= DB_EXCL; + h->db->set_pagesize(h->db, pagesize); + } + + ret = h->db->open(h->db, NULL, path, folder, DB_BTREE, createflags, 0600); + if (ret) { + h->db->close(h->db, 0); + h->env->close(h->env, 0); + mx_unlock_file(h->lockfile, h->fd, 0); + close(h->fd); + FREE(& h); + return NULL; + } + + return h; +} + +void +mutt_hcache_close(void *db) +{ + struct header_cache *h = db; + int ret; + + if (! h) { + return; + } + + h->db->close(h->db, 0); + h->env->close(h->env, 0); + mx_unlock_file(h->lockfile, h->fd, 0); + close(h->fd); + FREE(& h); +} + +void * +mutt_hcache_fetch(void *db, const char *filename, size_t (*keylen)(const char *fn)) +{ + DBT key; + DBT data; + struct header_cache *h = db; + + if (! h) { + return NULL; + } + + filename++; /* skip '/' */ + + mutt_hcache_dbt_init(&key, (void *) filename, keylen(filename)); + mutt_hcache_dbt_empty_init(&data); + data.flags = DB_DBT_MALLOC; + + h->db->get(h->db, NULL, &key, &data, 0); + + if (! crc32_matches(data.data, h->crc)) { + free(data.data); + return NULL; + } + + return data.data; +} + +int +mutt_hcache_store(void *db, const char *filename, HEADER *header, unsigned long long uid_validity, size_t (*keylen)(const char *fn)) +{ + DBT key; + DBT data; + int ret; + struct header_cache *h = db; + + if (! h) { + return -1; + } + + filename++; /* skip '/' */ + + mutt_hcache_dbt_init(&key, (void *) filename, keylen(filename)); + + mutt_hcache_dbt_empty_init(&data); + data.flags = DB_DBT_USERMEM; + data.data = mutt_hcache_dump(db, header, (signed int *) &data.size, uid_validity); + data.ulen = data.size; + + ret = h->db->put(h->db, NULL, &key, &data, 0); + + FREE(& data.data); + + return ret; +} + +int +mutt_hcache_delete(void *db, const char *filename, size_t (*keylen)(const char *fn)) +{ + DBT key; + struct header_cache *h = db; + + if (! h) { + return -1; + } + + filename++; /* skip '/' */ + + mutt_hcache_dbt_init(&key, (void *) filename, keylen(filename)); + return h->db->del(h->db, NULL, &key, 0); +} +#endif diff -Nru a/imap/imap.c b/imap/imap.c --- a/imap/imap.c 2004-08-18 09:43:27 +02:00 +++ b/imap/imap.c 2004-08-17 00:34:10 +02:00 @@ -602,6 +602,17 @@ if ((pc = imap_get_flags (&(idata->flags), pc)) == NULL) goto fail; } +#ifdef USE_HCACHE + /* save UIDVALIDITY for the header cache */ + else if (ascii_strncasecmp("OK [UIDVALIDITY", pc, 14) == 0) + { + dprint(2, (debugfile, "Getting mailbox UIDVALIDITY\n")); + pc += 3; + pc = imap_next_word(pc); + + sscanf(pc, "%u", &(idata->uid_validity)); + } +#endif else { pc = imap_next_word (pc); diff -Nru a/imap/imap_private.h b/imap/imap_private.h --- a/imap/imap_private.h 2004-08-18 09:43:27 +02:00 +++ b/imap/imap_private.h 2004-07-24 12:39:31 +02:00 @@ -179,6 +179,9 @@ unsigned int newMailCount; IMAP_CACHE cache[IMAP_CACHE_LEN]; int noclose : 1; +#ifdef USE_HCACHE + unsigned long long uid_validity; +#endif /* all folder flags - system flags AND keywords */ LIST *flags; diff -Nru a/imap/message.c b/imap/message.c --- a/imap/message.c 2004-08-18 09:43:27 +02:00 +++ b/imap/message.c 2004-08-06 02:09:41 +02:00 @@ -39,6 +39,12 @@ static int msg_parse_fetch (IMAP_HEADER* h, char* s); static char* msg_parse_flags (IMAP_HEADER* h, char* s); +#if USE_HCACHE +static int msg_fetch_header_fetch (CONTEXT* ctx, IMAP_HEADER* h, char* buf, + FILE* fp); +static size_t imap_hcache_keylen (const char *fn); +#endif /* USE_HCACHE */ + /* imap_read_headers: * Changed to read many headers instead of just one. It will return the * msgno of the last message read. It will return a value other than @@ -57,8 +63,18 @@ int fetchlast = 0; const char *want_headers = "DATE FROM SUBJECT TO CC MESSAGE-ID REFERENCES CONTENT-TYPE IN-REPLY-TO REPLY-TO LINES X-LABEL"; +#if USE_HCACHE + void *hc = NULL; + unsigned long long *uid_validity = NULL; + char uid_buf[64]; +#endif /* USE_HCACHE */ + ctx = idata->ctx; +#if USE_HCACHE + hc = mutt_hcache_open (HeaderCache, ctx->path); +#endif /* USE_HCACHE */ + if (mutt_bit_isset (idata->capabilities,IMAP4REV1)) { snprintf (hdrreq, sizeof (hdrreq), "BODY.PEEK[HEADER.FIELDS (%s)]", @@ -73,6 +89,9 @@ { /* Unable to fetch headers for lower versions */ mutt_error _("Unable to fetch headers from this IMAP server version."); mutt_sleep (2); /* pause a moment to let the user see the error */ +#if USE_HCACHE + mutt_hcache_close (hc); +#endif /* USE_HCACHE */ return -1; } @@ -83,6 +102,9 @@ { mutt_error (_("Could not create temporary file %s"), tempfile); mutt_sleep (2); +#if USE_HCACHE + mutt_hcache_close (hc); +#endif /* USE_HCACHE */ return -1; } unlink (tempfile); @@ -95,14 +117,88 @@ idata->reopen &= ~IMAP_NEWMAIL_PENDING; idata->newMailCount = 0; +#if USE_HCACHE + snprintf (buf, sizeof (buf), + "FETCH %d:%d (UID FLAGS)", msgbegin + 1, msgend + 1); + fetchlast = msgend + 1; + + imap_cmd_start (idata, buf); + + for (msgno = msgbegin; msgno <= msgend ; msgno++) + { + if (ReadInc && (!msgno || ((msgno+1) % ReadInc == 0))) + mutt_message (_("Evaluating cache... [%d/%d]"), msgno + 1, + msgend + 1); + + rewind (fp); + memset (&h, 0, sizeof (h)); + h.data = safe_calloc (1, sizeof (IMAP_HEADER_DATA)); + do + { + mfhrc = 0; + + rc = imap_cmd_step (idata); + if (rc != IMAP_CMD_CONTINUE) + break; + + if ((mfhrc = msg_fetch_header_fetch (idata->ctx, &h, idata->cmd.buf, fp)) == -1) + continue; + else if (mfhrc < 0) + break; + + /* make sure we don't get remnants from older larger message headers */ + fputs ("\n\n", fp); + + sprintf(uid_buf, "/%u", h.data->uid); /* XXX --tg 21:41 04-07-11 */ + uid_validity = (unsigned long long *) mutt_hcache_fetch (hc, uid_buf, &imap_hcache_keylen); + + if (uid_validity != NULL + && *uid_validity == idata->uid_validity) { + ctx->hdrs[msgno] = mutt_hcache_restore((unsigned char *) uid_validity, 0); + ctx->hdrs[msgno]->index = h.sid - 1; + if (h.sid != ctx->msgcount + 1) + dprint (1, (debugfile, "imap_read_headers: msgcount and sequence ID are inconsistent!")); + /* messages which have not been expunged are ACTIVE (borrowed from mh + * folders) */ + ctx->hdrs[msgno]->active = 1; + ctx->hdrs[msgno]->read = h.read; + ctx->hdrs[msgno]->old = h.old; + ctx->hdrs[msgno]->deleted = h.deleted; + ctx->hdrs[msgno]->flagged = h.flagged; + ctx->hdrs[msgno]->replied = h.replied; + ctx->hdrs[msgno]->changed = h.changed; + /* ctx->hdrs[msgno]->received is restored from mutt_hcache_restore */ + ctx->hdrs[msgno]->data = (void *) (h.data); + + ctx->msgcount++; + } + rewind (fp); + + FREE(&uid_validity); + + } + while ((rc != IMAP_CMD_OK) && ((mfhrc == -1) || + ((msgno + 1) >= fetchlast))); + } + + fetchlast = msgbegin; +#endif /* USE_HCACHE */ + for (msgno = msgbegin; msgno <= msgend ; msgno++) { if (ReadInc && (!msgno || ((msgno+1) % ReadInc == 0))) mutt_message (_("Fetching message headers... [%d/%d]"), msgno + 1, msgend + 1); + if (ctx->hdrs[msgno]) + continue; + if (msgno + 1 > fetchlast) { + fetchlast = msgno + 1; + while((fetchlast <= msgend) && (! ctx->hdrs[fetchlast])) + fetchlast++; + /* * Make one request for everything. This makes fetching headers an * order of magnitude faster if you have a large mailbox. @@ -112,11 +208,9 @@ */ snprintf (buf, sizeof (buf), "FETCH %d:%d (UID FLAGS INTERNALDATE RFC822.SIZE %s)", msgno + 1, - msgend + 1, hdrreq); + fetchlast, hdrreq); imap_cmd_start (idata, buf); - - fetchlast = msgend + 1; } /* freshen fp, h */ @@ -170,6 +264,11 @@ /* content built as a side-effect of mutt_read_rfc822_header */ ctx->hdrs[msgno]->content->length = h.content_length; +#if USE_HCACHE + sprintf(uid_buf, "/%u", h.data->uid); + mutt_hcache_store(hc, uid_buf, ctx->hdrs[msgno], idata->uid_validity, &imap_hcache_keylen); +#endif /* USE_HCACHE */ + ctx->msgcount++; } while ((rc != IMAP_CMD_OK) && ((mfhrc == -1) || @@ -179,7 +278,9 @@ { imap_free_header_data ((void**) &h.data); fclose (fp); - +#if USE_HCACHE + mutt_hcache_close (hc); +#endif /* USE_HCACHE */ return -1; } @@ -194,6 +295,10 @@ } } +#if USE_HCACHE + mutt_hcache_close (hc); +#endif /* USE_HCACHE */ + fclose(fp); if (ctx->msgcount > oldmsgcount) @@ -724,6 +829,7 @@ return s; } + /* msg_fetch_header: import IMAP FETCH response into an IMAP_HEADER. * Expects string beginning with * n FETCH. * Returns: @@ -782,6 +888,56 @@ return rc; } + +#if USE_HCACHE +static size_t imap_hcache_keylen (const char *fn) +{ + return mutt_strlen(fn); +} + +/* msg_fetch_header: import IMAP FETCH response into an IMAP_HEADER. + * Expects string beginning with * n FETCH. + * Returns: + * 0 on success + * -1 if the string is not a fetch response + * -2 if the string is a corrupt fetch response */ +static int msg_fetch_header_fetch (CONTEXT* ctx, IMAP_HEADER* h, char* buf, FILE* fp) +{ + IMAP_DATA* idata; + long bytes; + int rc = -1; /* default now is that string isn't FETCH response*/ + + idata = (IMAP_DATA*) ctx->data; + + if (buf[0] != '*') + return rc; + + /* skip to message number */ + buf = imap_next_word (buf); + h->sid = atoi (buf); + + /* find FETCH tag */ + buf = imap_next_word (buf); + if (ascii_strncasecmp ("FETCH", buf, 5)) + return rc; + + rc = -2; /* we've got a FETCH response, for better or worse */ + if (!(buf = strchr (buf, '('))) + return rc; + buf++; + + if (msg_parse_fetch (h, buf) < 0) { + return -2; + } + + if (!(buf = strchr (buf, ')'))) + return rc; + buf++; + + return 0; +} +#endif /* USE_HCACHE */ + /* msg_has_flag: do a caseless comparison of the flag against a flag list, * return 1 if found or flag list has '\*', 0 otherwise */ diff -Nru a/init.h b/init.h --- a/init.h 2004-08-18 09:43:12 +02:00 +++ b/init.h 2004-09-20 22:58:42 +02:00 @@ -981,6 +981,34 @@ ** \fBDON'T CHANGE THIS SETTING UNLESS YOU ARE REALLY SURE WHAT YOU ARE ** DOING!\fP */ +#if USE_HCACHE + + { "header_cache", DT_PATH, R_NONE, UL &HeaderCache, 0 }, + /* + ** .pp + ** The header_cache variable points to the header cache database. If + ** header_cache points to a directory there will be created one header cache + ** database per folder within this directory. If it doesn't point to a directory a + ** global header cache for all folders is used. Per default it is unset and so + ** no header caching will be used. + */ + { "maildir_header_cache_verify", DT_BOOL, R_NONE, OPTHCACHEVERIFY, 1 }, + /* + ** .pp + ** Check for Maildir unaware programs other than mutt having modified maildir + ** files when the header cache is in use. This incurs one stat(2) per + ** message every time the folder is opened. + */ + { "header_cache_pagesize", DT_STR, R_NONE, UL &HeaderCachePageSize, UL "16384" }, + /* + ** .pp + ** Change the maildir header cache database page size. Too large + ** or too small of a page size for the common header can waste + ** space, memory effectiveness, or CPU time. The default should be more or + ** less the best you can get. For details google after mutt maildir header + ** cache (first hit). + */ +#endif /* USE_HCACHE */ { "maildir_trash", DT_BOOL, R_NONE, OPTMAILDIRTRASH, 0 }, /* ** .pp diff -Nru a/main.c b/main.c --- a/main.c 2004-08-18 09:43:12 +02:00 +++ b/main.c 2004-07-24 12:39:30 +02:00 @@ -411,6 +411,12 @@ "-HAVE_GETADDRINFO " #endif +#if USE_HCACHE + "+USE_HCACHE " +#else + "-USE_HCACHE " +#endif + ); #ifdef ISPELL diff -Nru a/mh.c b/mh.c --- a/mh.c 2004-08-18 09:43:13 +02:00 +++ b/mh.c 2004-09-03 14:33:12 +02:00 @@ -779,30 +779,79 @@ return r; } +#if USE_HCACHE +static size_t maildir_hcache_keylen (const char *fn) +{ + const char * p = strchr (fn, ':'); + return p ? (size_t) (p - fn) : mutt_strlen(fn); +} +#endif /* * This function does the second parsing pass for a maildir-style * folder. */ - void maildir_delayed_parsing (CONTEXT * ctx, struct maildir *md) { struct maildir *p; char fn[_POSIX_PATH_MAX]; + int count; - for (p = md; p; p = p->next) - if (p && p->h && !p->header_parsed) - { - snprintf (fn, sizeof (fn), "%s/%s", ctx->path, p->h->path); - if (maildir_parse_message (ctx->magic, fn, p->h->old, p->h)) - p->header_parsed = 1; - else - mutt_free_header (&p->h); - } +#if USE_HCACHE + void *hc = NULL; + void *data; + unsigned int size; + struct timeval *when = NULL; + struct stat lastchanged; + int ret; + + hc = mutt_hcache_open (HeaderCache, ctx->path); +#endif + + for (p = md, count = 0; p; p = p->next, count++) + { + if (! (p && p->h && !p->header_parsed)) + continue; + +#if USE_HCACHE + data = mutt_hcache_fetch (hc, p->h->path + 3, &maildir_hcache_keylen); + when = (struct timeval *) data; +#endif + + snprintf (fn, sizeof (fn), "%s/%s", ctx->path, p->h->path); + +#if USE_HCACHE + if (option(OPTHCACHEVERIFY)) { + ret = stat(fn, &lastchanged); + } else { + lastchanged.st_mtime = 0; + ret = 0; + } + + if (data != NULL && !ret && lastchanged.st_mtime <= when->tv_sec) + { + p->h = mutt_hcache_restore ((unsigned char *)data, &p->h); + maildir_parse_flags (p->h, fn); + } else +#endif + if (maildir_parse_message (ctx->magic, fn, p->h->old, p->h)) + { + p->header_parsed = 1; + maildir_parse_flags (p->h, fn); +#if USE_HCACHE + mutt_hcache_store (hc, p->h->path + 3, p->h, 0, &maildir_hcache_keylen); +#endif + } else + mutt_free_header (&p->h); +#if USE_HCACHE + FREE(&data); +#endif + } +#if USE_HCACHE + mutt_hcache_close (hc); +#endif } - - /* Read a MH/maildir style mailbox. * * args: @@ -1293,6 +1342,9 @@ { char path[_POSIX_PATH_MAX], tmp[_POSIX_PATH_MAX]; int i, j; +#if USE_HCACHE + void *hc = NULL; +#endif /* USE_HCACHE */ if (ctx->magic == M_MH) i = mh_check_mailbox (ctx, index_hint); @@ -1302,6 +1354,11 @@ if (i != 0) return i; +#if USE_HCACHE + if (ctx->magic == M_MAILDIR) + hc = mutt_hcache_open(HeaderCache, ctx->path); +#endif /* USE_HCACHE */ + for (i = 0; i < ctx->msgcount; i++) { if (ctx->hdrs[i]->deleted @@ -1310,7 +1367,13 @@ snprintf (path, sizeof (path), "%s/%s", ctx->path, ctx->hdrs[i]->path); if (ctx->magic == M_MAILDIR || (option (OPTMHPURGE) && ctx->magic == M_MH)) + { +#if USE_HCACHE + if (ctx->magic == M_MAILDIR) + mutt_hcache_delete (hc, ctx->hdrs[i]->path + 3, &maildir_hcache_keylen); +#endif /* USE_HCACHE */ unlink (path); + } else if (ctx->magic == M_MH) { /* MH just moves files out of the way when you delete them */ @@ -1332,16 +1395,21 @@ if (ctx->magic == M_MAILDIR) { if (maildir_sync_message (ctx, i) == -1) - return -1; + goto err; } else { if (mh_sync_message (ctx, i) == -1) - return -1; + goto err; } } } +#if USE_HCACHE + if (ctx->magic == M_MAILDIR) + mutt_hcache_close (hc); +#endif /* USE_HCACHE */ + if (ctx->magic == M_MH) mh_update_sequences (ctx); @@ -1362,6 +1430,13 @@ } return 0; + +err: +#if USE_HCACHE + if (ctx->magic == M_MAILDIR) + mutt_hcache_close (hc); +#endif /* USE_HCACHE */ + return -1; } static char *maildir_canon_filename (char *dest, const char *src, size_t l) diff -Nru a/mutt.h b/mutt.h --- a/mutt.h 2004-08-18 09:43:13 +02:00 +++ b/mutt.h 2004-07-24 12:39:30 +02:00 @@ -345,6 +345,9 @@ OPTFORCENAME, OPTFORWDECODE, OPTFORWQUOTE, +#if USE_HCACHE + OPTHCACHEVERIFY, +#endif OPTHDRS, OPTHEADER, OPTHELP, diff -Nru a/protos.h b/protos.h --- a/protos.h 2004-08-18 09:43:16 +02:00 +++ b/protos.h 2004-08-09 08:05:48 +02:00 @@ -99,6 +99,16 @@ ENVELOPE *mutt_read_rfc822_header (FILE *, HEADER *, short, short); HEADER *mutt_dup_header (HEADER *); +#if USE_HCACHE +void *mutt_hcache_open(const char *path, const char *folder); +void mutt_hcache_close(void *db); +HEADER *mutt_hcache_restore(const unsigned char *d, HEADER **oh); +void *mutt_hcache_fetch(void *db, const char *filename, size_t (*keylen)(const char *fn)); +int mutt_hcache_store(void *db, const char *filename, HEADER *h, unsigned long long uid_validity, size_t (*keylen)(const char *fn)); +int mutt_hcache_delete(void *db, const char *filename, size_t (*keylen)(const char *fn)); +#endif /* USE_HCACHE */ + + ATTACHPTR **mutt_gen_attach_list (BODY *, int, ATTACHPTR **, short *, short *, int, int); time_t mutt_decrease_mtime (const char *, struct stat *);