memcache support
authorMatthias Förste <foerste@schlittermann.de>
Tue, 11 Aug 2009 11:51:15 +0200
changeset 3 c1c63bcbc84f
parent 2 21c93a4387f7
child 4 5d6a10b5469c
memcache support
exim4/4.69/memcache-support.patch
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/exim4/4.69/memcache-support.patch	Tue Aug 11 11:51:15 2009 +0200
@@ -0,0 +1,482 @@
+diff -r 2986669e93b0 -r c0445d21ca67 debian/EDITME.exim4-light.diff
+--- a/debian/EDITME.exim4-light.diff	Mon Aug 10 16:00:12 2009 +0200
++++ b/debian/EDITME.exim4-light.diff	Mon Aug 10 16:01:25 2009 +0200
+@@ -1,5 +1,5 @@
+---- /tmp/dpep-work.wG8120/trunk/build-tree/src/EDITME	2007-07-17 11:04:08.000000000 +0200
+-+++ EDITME.exim4-light	2007-07-17 11:04:27.000000000 +0200
++--- /home/luser/src/debian-packages/exim4-4.69/build-tree/src/EDITME	2009-08-10 15:37:12.000000000 +0200
+++++ EDITME.exim4-light	2009-08-10 15:45:37.000000000 +0200
+ @@ -100,7 +100,7 @@
+  # /usr/local/sbin. The installation script will try to create this directory,
+  # and any superior directories, if they do not exist.
+@@ -196,9 +196,15 @@
+  
+  
+  #------------------------------------------------------------------------------
+-@@ -1149,3 +1156,6 @@
++@@ -1148,4 +1155,12 @@
++ 
+  # ENABLE_DISABLE_FSYNC=yes
+  
+++#------------------------------------------------------------------------------
+++# support storage/retrieval of values to and from memcache servers via
+++# expansions
+++SUPPORT_MEMCACHE=yes
+++
+  # End of EDITME for Exim 4.
+ +
+ +# enable IPv6 support
+diff -r 2986669e93b0 -r c0445d21ca67 debian/patches/00list
+--- a/debian/patches/00list	Mon Aug 10 16:00:12 2009 +0200
++++ b/debian/patches/00list	Mon Aug 10 16:01:25 2009 +0200
+@@ -12,3 +12,4 @@
+ 60_convert4r4
+ 66_enlarge-dh-parameters-size
+ 70_remove_exim-users_references
++80_memcache
+diff -r 2986669e93b0 -r c0445d21ca67 debian/patches/80_memcache.dpatch
+--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
++++ b/debian/patches/80_memcache.dpatch	Mon Aug 10 16:01:25 2009 +0200
+@@ -0,0 +1,442 @@
++#! /bin/sh /usr/share/dpatch/dpatch-run
++## 80_memcache.dpatch by  <luser@lenny.vbox.hurz.is.schlittermann.de>
++##
++## All lines beginning with `## DP:' are a description of the patch.
++## DP: key/value storage/retrieval via memcache
++
++@DPATCH@
++
++diff -r 6c0f7fa36c53 OS/Makefile-Base
++--- a/build-tree/OS/Makefile-Base	Wed Jul 29 15:15:06 2009 +0200
+++++ b/build-tree/OS/Makefile-Base	Mon Aug 10 10:31:21 2009 +0200
++@@ -543,7 +543,7 @@
++ dns.o:           $(HDRS) dns.c
++ enq.o:           $(HDRS) enq.c
++ exim.o:          $(HDRS) exim.c
++-expand.o:        $(HDRS) expand.c
+++expand.o:        $(HDRS) memcache.h expand.c
++ filter.o:        $(HDRS) filter.c
++ filtertest.o:    $(HDRS) filtertest.c
++ globals.o:       $(HDRS) globals.c
++diff -r 6c0f7fa36c53 doc/OptionLists.txt
++--- a/build-tree/doc/OptionLists.txt	Wed Jul 29 15:15:06 2009 +0200
+++++ b/build-tree/doc/OptionLists.txt	Mon Aug 10 10:31:21 2009 +0200
++@@ -11,7 +11,7 @@
++   4. Those that can appear in the build time configuration for the Exim monitor
++      (Local/eximon.conf).
++ 
++-This file was last updated for Exim release 4.67.
+++This file was last updated for Exim release 4.69.
++ 
++ 
++ 1. RUN TIME OPTIONS
++@@ -887,6 +887,7 @@
++ SUPPORT_CRYPTEQ              optional     support crypteq (if no auths)
++ SUPPORT_MAILDIR              optional     support for maildir delivery
++ SUPPORT_MAILSTORE            optional     support for mailstore delivery
+++SUPPORT_MEMCACHE             optional*    support for memcache storage and retrieval
++ SUPPORT_MBX                  optional     support for MBX delivery
++ SUPPORT_MOVE_FROZEN_MESSAGES optional*    support for frozen message moving
++ SUPPORT_PAM                  optional     support for PAM authentication
++diff -r 6c0f7fa36c53 doc/memcache.txt
++--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++++ b/build-tree/doc/memcache.txt	Mon Aug 10 10:31:21 2009 +0200
++@@ -0,0 +1,4 @@
+++store some value under key (expands to 'true' if successful):
+++${memcache{memcache_servers}{set}{namespace}{key}{value}{expiration}{timeout}}
+++lookup the value of some key (expands to 'true' if the key has been found; stores the value in $value)
+++${memcache{memcache_servers}{get}{namespace}{key}{$value}{timeout}}
++diff -r 6c0f7fa36c53 scripts/MakeLinks
++--- a/build-tree/scripts/MakeLinks	Wed Jul 29 15:15:06 2009 +0200
+++++ b/build-tree/scripts/MakeLinks	Mon Aug 10 10:31:21 2009 +0200
++@@ -200,6 +200,7 @@
++ ln -s ../src/globals.h         globals.h
++ ln -s ../src/local_scan.h      local_scan.h
++ ln -s ../src/macros.h          macros.h
+++ln -s ../src/memcache.h        memcache.h
++ ln -s ../src/mytypes.h         mytypes.h
++ ln -s ../src/osfunctions.h     osfunctions.h
++ ln -s ../src/store.h           store.h
++diff -r 6c0f7fa36c53 src/config.h.defaults
++--- a/build-tree/src/config.h.defaults	Wed Jul 29 15:15:06 2009 +0200
+++++ b/build-tree/src/config.h.defaults	Mon Aug 10 10:31:21 2009 +0200
++@@ -120,6 +120,7 @@
++ #define SUPPORT_MAILDIR
++ #define SUPPORT_MAILSTORE
++ #define SUPPORT_MBX
+++#define SUPPORT_MEMCACHE
++ #define SUPPORT_MOVE_FROZEN_MESSAGES
++ #define SUPPORT_PAM
++ #define SUPPORT_TLS
++diff -r 6c0f7fa36c53 src/expand.c
++--- a/build-tree/src/expand.c	Wed Jul 29 15:15:06 2009 +0200
+++++ b/build-tree/src/expand.c	Mon Aug 10 10:31:21 2009 +0200
++@@ -25,6 +25,10 @@
++ 
++ #ifdef LOOKUP_LDAP
++ #include "lookups/ldap.h"
+++#endif
+++
+++#ifdef SUPPORT_MEMCACHE
+++#include "memcache.h"
++ #endif
++ 
++ #ifdef SUPPORT_CRYPTEQ
++@@ -113,6 +117,7 @@
++   US"length",
++   US"lookup",
++   US"map",
+++  US"memcache",
++   US"nhash",
++   US"perl",
++   US"prvs",
++@@ -135,6 +140,7 @@
++   EITEM_LENGTH,
++   EITEM_LOOKUP,
++   EITEM_MAP,
+++  EITEM_MEMCACHE,
++   EITEM_NHASH,
++   EITEM_PERL,
++   EITEM_PRVS,
++@@ -4980,6 +4986,229 @@
++         }
++       }
++     #endif /* EXPAND_DLFUNC */
+++
+++    case EITEM_MEMCACHE:
+++    #ifndef SUPPORT_MEMCACHE
+++    expand_string_message = US"\"${memcache\" encountered, but this facility "
+++      "is not included in this binary";
+++    goto EXPAND_FAILED;
+++
+++    #else   /* SUPPORT_MEMCACHE */
+++    /* we dont use flags right now - should we? */
+++
+++      {
+++        uschar *memcache_timeout = US"0s";   /* with time unit postfix, fex 7d */
+++        uschar *memcache_expiration = US"0"; /* in seconds */
+++        int i_memcache_expiration = -1, i_memcache_timeout = -1;
+++        int memcache_operation;
+++        uschar *sub_arg[7];
+++
+++        uschar *memcache_servers[MEMCACHE_SERVERS_MAX];
+++        tree_node *memcache_hosts_node;
+++        uschar *memcache_hosts_string, *memcache_op_string;
+++        uschar *memcache_host;
+++        int memcache_hostlist_separator = 0;
+++        int i_memcache_server = 0;
+++        int n_memcache_servers = 0;
+++
+++        uschar *memcache_request;
+++        int memcache_request_len;
+++
+++        uschar *memcache_readsocket_result;
+++
+++        if ((expand_forbid & RDO_MEMCACHE) != 0)
+++          {
+++          expand_string_message = US"memcache storage/retrieval are not permitted";
+++          goto EXPAND_FAILED;
+++          }
+++
+++        /* read up to 7 args */
+++        switch(read_subs(sub_arg, 7, 5, &s, skipping, TRUE, US"memcache"))
+++          {
+++          case 1: goto EXPAND_FAILED_CURLY;
+++          case 2:                             /* Won't occur: no end check */
+++          case 3: goto EXPAND_FAILED;
+++          }
+++
+++        /* validate operation and set argument indices */
+++        memcache_op_string = string_sprintf(US"%S", sub_arg[1]);
+++        if (Ustrcmp(memcache_op_string, US"set") == 0)
+++          {
+++          i_memcache_expiration = 5; i_memcache_timeout = 6; memcache_operation = MEMCACHE_OP_SET;
+++          }
+++        else if (Ustrcmp(memcache_op_string, US"get") == 0)
+++          {
+++          i_memcache_timeout = 5; memcache_operation = MEMCACHE_OP_GET;
+++          }
+++        else
+++          {
+++          expand_string_message = string_sprintf(US"invalid memcache operation '%s'",
+++            sub_arg[1]);
+++          goto EXPAND_FAILED;
+++          }
+++
+++        /* expiration */
+++        if (i_memcache_expiration >=0 && sub_arg[i_memcache_expiration] != NULL)
+++          {
+++          memcache_expiration = string_sprintf(US"%d", readconf_readtime(sub_arg[i_memcache_expiration], 0, FALSE));
+++          if (memcache_expiration < 0)
+++            {
+++            expand_string_message = string_sprintf(US"bad time value %s",
+++              sub_arg[i_memcache_expiration]);
+++            goto EXPAND_FAILED;
+++            }
+++          }
+++
+++        /* now that we have an expiration time we should be able to build the request string */
+++        if (memcache_operation == MEMCACHE_OP_SET)
+++          {
+++          uschar* data = sub_arg[4];
+++          int data_len = Ustrlen(data);
+++
+++          /* writeop [namespace]key flags expiration size-in-bytes [noreply] */
+++          uschar *cmd = string_sprintf(US"%s %s%s %d %s %d %s", 
+++              US"set",
+++              sub_arg[2],
+++              sub_arg[3],
+++              0,
+++              memcache_expiration,
+++              data_len,
+++              US"");
+++          int cmd_len = Ustrlen(cmd);
+++
+++          memcache_request = string_sprintf(US"%s\r\n%s\r\n", cmd, data);
+++          memcache_request_len = Ustrlen(memcache_request);
+++          }
+++        else if (memcache_operation == MEMCACHE_OP_GET)
+++          {
+++          memcache_request = string_sprintf(US"%s %s%s\r\n", US"get", sub_arg[2], sub_arg[3]);
+++          memcache_request_len = Ustrlen(memcache_request);
+++          }
+++        else
+++          {
+++          expand_string_message = string_sprintf(US"invalid memcache operation: '%s'", memcache_op_string);
+++          goto EXPAND_FAILED;
+++          }
+++
+++        /* memcache servers - we expect either a named list or just a list */
+++        if (sub_arg[0][0] == '+')
+++          {
+++          if (NULL == (memcache_hosts_node = tree_search(hostlist_anchor, sub_arg[0] + sizeof(uschar))))
+++            {
+++            expand_string_message = string_sprintf(US"named list '%s' not found",
+++              sub_arg[0]);
+++            goto EXPAND_FAILED;
+++            }
+++            memcache_hosts_string = ((namedlist_block *) memcache_hosts_node->data.ptr)->string;
+++          }
+++        else
+++          {
+++            memcache_hosts_string = sub_arg[0];
+++          }
+++
+++        while ((NULL != (memcache_host = string_nextinlist(&memcache_hosts_string, &memcache_hostlist_separator, NULL, 0))))
+++          {
+++          if (i_memcache_server >= MEMCACHE_SERVERS_MAX)
+++            {
+++            expand_string_message = string_sprintf(US"too many memcache servers, %d is max",
+++              MEMCACHE_SERVERS_MAX);
+++            goto EXPAND_FAILED;
+++            }
+++          memcache_servers[i_memcache_server] = memcache_host;
+++          i_memcache_server++;
+++          }
+++        n_memcache_servers = i_memcache_server;
+++
+++        /* talk to server(s) unless we are skipping */
+++        if (!skipping)
+++          {
+++          for (i_memcache_server = 0; i_memcache_server < n_memcache_servers; 
+++                 i_memcache_server++)
+++            {
+++
+++            uschar *memcache_server_name = memcache_servers[i_memcache_server];
+++            uschar *memcache_port_name = Ustrrchr(memcache_server_name, ':');
+++            uschar *memcache_default_port_name = US"11211";
+++
+++            /* Sort out the port */
+++            if (memcache_port_name == NULL)
+++              {
+++              memcache_port_name = memcache_default_port_name;
+++              }
+++            else
+++              {
+++              *memcache_port_name++ = 0;           /* Terminate server name */
+++              }
+++
+++            uschar *memcache_server_readsocket =
+++              string_sprintf(US"${readsocket{inet:%s:%s}{%s}{%s}}",
+++              memcache_server_name,
+++              memcache_port_name,
+++              memcache_request,
+++              sub_arg[i_memcache_timeout] == NULL ? memcache_timeout : sub_arg[i_memcache_timeout]);
+++            DEBUG(D_memcache)
+++              {
+++              debug_printf("readsocket expansion item: '%s'\n", memcache_server_readsocket);
+++              }
+++            memcache_readsocket_result = expand_string(memcache_server_readsocket);
+++            if (memcache_readsocket_result != NULL) 
+++              {
+++              if (memcache_operation == MEMCACHE_OP_GET
+++                  && 0 == Ustrncmp(memcache_readsocket_result, US"VALUE ", Ustrlen(US"VALUE ")))
+++                {
+++                /* we tried to use expand_gettokened but it segfaults and we
+++                   didnt find out why */
+++                uschar *memcache_value_size = memcache_readsocket_result;
+++                int i;
+++                for (i = 0; i < 3; i++) { memcache_value_size = Ustrchr(memcache_value_size, ' ') + 1; }
+++
+++                lookup_value = strstr(memcache_readsocket_result, "\r\n") + Ustrlen("\r\n");
+++                lookup_value[
+++                  Ustrtol(memcache_value_size,
+++                      NULL,
+++                      0)] = 0;
+++
+++                DEBUG(D_memcache) { debug_printf("lookup_value: '%s'\n", lookup_value); }
+++
+++                }
+++              else if (memcache_operation == MEMCACHE_OP_SET
+++                  && 0 == Ustrcmp(memcache_readsocket_result, US"STORED\r\n"))
+++                {
+++                }
+++              else
+++                {
+++                /* something went wrong - try something else */
+++                DEBUG(D_memcache)
+++                  {
+++                  debug_printf("memcache operation '%s' failed: '%s'\n",
+++                      memcache_op_string,
+++                      memcache_readsocket_result);
+++                  }
+++                continue;
+++                }
+++
+++              yield = string_cat(yield,
+++                &size,
+++                &ptr,
+++                US"yes",
+++                Ustrlen(US"yes"));
+++              break; 
+++              }
+++
+++            }
+++
+++            if (memcache_readsocket_result == NULL) 
+++              {
+++              /* every try failed if we come here */
+++              expand_string_message = 
+++                string_sprintf(US"all memcache servers failed - last error was: '%s'",
+++                expand_string_message);
+++              goto EXPAND_FAILED;
+++              }
+++          }
+++        continue;
+++      }
+++    #endif /* SUPPORT_MEMCACHE */
++     }
++ 
++   /* Control reaches here if the name is not recognized as one of the more
++diff -r 6c0f7fa36c53 src/globals.c
++--- a/build-tree/src/globals.c	Wed Jul 29 15:15:06 2009 +0200
+++++ b/build-tree/src/globals.c	Mon Aug 10 10:31:21 2009 +0200
++@@ -448,6 +448,7 @@
++   { US"load",           D_load },
++   { US"local_scan",     D_local_scan },
++   { US"lookup",         D_lookup },
+++  { US"memcache",       D_memcache },
++   { US"memory",         D_memory },
++   { US"pid",            D_pid },
++   { US"process_info",   D_process_info },
++diff -r 6c0f7fa36c53 src/macros.h
++--- a/build-tree/src/macros.h	Wed Jul 29 15:15:06 2009 +0200
+++++ b/build-tree/src/macros.h	Mon Aug 10 10:31:21 2009 +0200
++@@ -313,19 +313,20 @@
++ #define D_load                       0x00008000
++ #define D_lookup                     0x00010000
++ #define D_memory                     0x00020000
++-#define D_pid                        0x00040000
++-#define D_process_info               0x00080000
++-#define D_queue_run                  0x00100000
++-#define D_receive                    0x00200000
++-#define D_resolver                   0x00400000
++-#define D_retry                      0x00800000
++-#define D_rewrite                    0x01000000
++-#define D_route                      0x02000000
++-#define D_timestamp                  0x04000000
++-#define D_tls                        0x08000000
++-#define D_transport                  0x10000000
++-#define D_uid                        0x20000000
++-#define D_verify                     0x40000000
+++#define D_memcache                   0x00040000
+++#define D_pid                        0x00080000
+++#define D_process_info               0x00100000
+++#define D_queue_run                  0x00200000
+++#define D_receive                    0x00400000
+++#define D_resolver                   0x00800000
+++#define D_retry                      0x01000000
+++#define D_rewrite                    0x02000000
+++#define D_route                      0x04000000
+++#define D_timestamp                  0x08000000
+++#define D_tls                        0x10000000
+++#define D_transport                  0x20000000
+++#define D_uid                        0x40000000
+++#define D_verify                     0x80000000
++ 
++ /* The D_all value must always have all bits set, as it is recognized specially
++ by the function that decodes debug and log selectors. This is to enable it to
++@@ -515,21 +516,21 @@
++ #define RDO_INCLUDE      0x00000100  /* Forbid :include: */
++ #define RDO_LOG          0x00000200  /* Forbid "log" */
++ #define RDO_LOOKUP       0x00000400  /* Forbid "lookup" in expansion in filter */
++-#define RDO_PERL         0x00000800  /* Forbid "perl" in expansion in filter */
++-#define RDO_READFILE     0x00001000  /* Forbid "readfile" in exp in filter */
++-#define RDO_READSOCK     0x00002000  /* Forbid "readsocket" in exp in filter */
++-#define RDO_RUN          0x00004000  /* Forbid "run" in expansion in filter */
++-#define RDO_DLFUNC       0x00008000  /* Forbid "dlfunc" in expansion in filter */
++-#define RDO_REALLOG      0x00010000  /* Really do log (not testing/verifying) */
++-#define RDO_REWRITE      0x00020000  /* Rewrite generated addresses */
++-#define RDO_EXIM_FILTER  0x00040000  /* Forbid Exim filters */
++-#define RDO_SIEVE_FILTER 0x00080000  /* Forbid Sieve filters */
++-#define RDO_PREPEND_HOME 0x00100000  /* Prepend $home to relative paths in Exim filter save commands */
++-
+++#define RDO_MEMCACHE     0x00000800  /* Forbid "memcache" in expansion in filter */
+++#define RDO_PERL         0x00001000  /* Forbid "perl" in expansion in filter */
+++#define RDO_READFILE     0x00002000  /* Forbid "readfile" in exp in filter */
+++#define RDO_READSOCK     0x00004000  /* Forbid "readsocket" in exp in filter */
+++#define RDO_RUN          0x00008000  /* Forbid "run" in expansion in filter */
+++#define RDO_DLFUNC       0x00010000  /* Forbid "dlfunc" in expansion in filter */
+++#define RDO_REALLOG      0x00020000  /* Really do log (not testing/verifying) */
+++#define RDO_REWRITE      0x00040000  /* Rewrite generated addresses */
+++#define RDO_EXIM_FILTER  0x00080000  /* Forbid Exim filters */
+++#define RDO_SIEVE_FILTER 0x00100000  /* Forbid Sieve filters */
+++#define RDO_PREPEND_HOME 0x00200000  /* Prepend $home to relative paths in Exim filter save commands */
++ /* This is the set that apply to expansions in filters */
++ 
++ #define RDO_FILTER_EXPANSIONS \
++-  (RDO_EXISTS|RDO_LOOKUP|RDO_PERL|RDO_READFILE|RDO_READSOCK|RDO_RUN|RDO_DLFUNC)
+++  (RDO_EXISTS|RDO_LOOKUP|RDO_MEMCACHE|RDO_PERL|RDO_READFILE|RDO_READSOCK|RDO_RUN|RDO_DLFUNC)
++ 
++ /* As well as the RDO bits themselves, we need the bit numbers in order to
++ access (most of) the individual bits as separate options. This could be
++@@ -537,8 +538,9 @@
++ 
++ enum { RDON_BLACKHOLE, RDON_DEFER, RDON_EACCES, RDON_ENOTDIR, RDON_EXISTS,
++   RDON_FAIL, RDON_FILTER, RDON_FREEZE, RDON_INCLUDE, RDON_LOG, RDON_LOOKUP,
++-  RDON_PERL, RDON_READFILE, RDON_READSOCK, RDON_RUN, RDON_DLFUNC, RDON_REALLOG,
++-  RDON_REWRITE, RDON_EXIM_FILTER, RDON_SIEVE_FILTER, RDON_PREPEND_HOME };
+++  RDON_MEMCACHE, RDON_PERL, RDON_READFILE, RDON_READSOCK, RDON_RUN,
+++  RDON_DLFUNC, RDON_REALLOG, RDON_REWRITE, RDON_EXIM_FILTER, RDON_SIEVE_FILTER,
+++  RDON_PREPEND_HOME };
++ 
++ /* Results of filter or forward file processing. Some are only from a filter;
++ some are only from a forward file. */
++diff -r 6c0f7fa36c53 src/memcache.h
++--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++++ b/build-tree/src/memcache.h	Mon Aug 10 10:31:21 2009 +0200
++@@ -0,0 +1,14 @@
+++#ifndef MEMCACHE_H
+++
+++#define MEMCACHE_SERVERS_MAX 16
+++#define MEMCACHE_OP_SET 0
+++#define MEMCACHE_OP_GET 1
+++
+++/*
+++extern void memcache_addhost(uschar *, uschar *, void *);
+++*/
+++/* host list */
+++
+++/* put declarations before */
+++#define MEMCACHE_H
+++#endif