diff -Naur apache_1.3.6+01-06/conf/httpd.conf-dist apache_1.3.6+01-07/conf/httpd.conf-dist --- apache_1.3.6+01-06/conf/httpd.conf-dist Wed Mar 10 01:14:27 1999 +++ apache_1.3.6+01-07/conf/httpd.conf-dist Tue Jul 20 22:19:58 1999 @@ -179,6 +179,14 @@ #BindAddress * # +# SingleListen: Require that each child process listen to a single IP +# address and port, from the list specified using Listen options. +# Listen addresses are distributed evenly across child processes. This +# avoids the whole serialized accept problem. +# +SingleListen off + +# # Dynamic Shared Object (DSO) Support # # To be able to use the functionality of a module which was built as a DSO you diff -Naur apache_1.3.6+01-06/htdocs/manual/bind.html apache_1.3.6+01-07/htdocs/manual/bind.html --- apache_1.3.6+01-06/htdocs/manual/bind.html Mon Mar 22 16:17:33 1999 +++ apache_1.3.6+01-07/htdocs/manual/bind.html Tue Jul 20 22:21:25 1999 @@ -72,7 +72,7 @@ Syntax: Listen [ port | IP-address:port ]
+>Syntax: Listen [ port | IP-address:port ] [ CPU-num ]
Port directive. If an IP address is given as well as a port, the server -will listen on the given port and interface.

Multiple Listen +will listen on the given port and interface. + +

The optional CPU-num argument specifies a CPU to which child +processes listening to this address should be bound, when the SingleListen option is +on. + +

Multiple Listen directives may be used to specify a number of addresses and ports to listen to. The server will respond to requests from any of the listed addresses and ports.

@@ -131,6 +138,7 @@ Virtual Hosts, BindAddress directive, Port directive, +SingleListen directive, DNS Issues and <VirtualHost> section. diff -Naur apache_1.3.6+01-06/htdocs/manual/misc/perf-tuning.html apache_1.3.6+01-07/htdocs/manual/misc/perf-tuning.html --- apache_1.3.6+01-06/htdocs/manual/misc/perf-tuning.html Mon Mar 22 16:17:45 1999 +++ apache_1.3.6+01-07/htdocs/manual/misc/perf-tuning.html Tue Jul 20 22:24:35 1999 @@ -238,7 +238,7 @@ contains timing indications. For highest performance, set ExtendedStatus off (which is the default). -

accept Serialization - multiple sockets

+

accept Serialization - multiple sockets

This discusses a shortcoming in the Unix socket API. Suppose your diff -Naur apache_1.3.6+01-06/htdocs/manual/mod/core.html apache_1.3.6+01-07/htdocs/manual/mod/core.html --- apache_1.3.6+01-06/htdocs/manual/mod/core.html Thu Jul 8 12:05:03 1999 +++ apache_1.3.6+01-07/htdocs/manual/mod/core.html Tue Jul 20 22:26:34 1999 @@ -92,6 +92,7 @@

  • ServerSignature
  • ServerTokens
  • ServerType +
  • SingleListen
  • StartServers
  • ThreadsPerChild
  • TimeOut @@ -1575,7 +1576,7 @@ HREF="directive-dict.html#Syntax" REL="Help" >Syntax: -Listen [IP address:]port number
    +Listen [IP address:]port number [CPU-num]
    Compatibility: Listen is only available in Apache -1.1 and later.

    +1.1 and later. +The optional CPU-num argument is only available in Apache 1.3.7 and later.

    The Listen directive instructs Apache to listen to more than one IP address or port; by default it responds to requests on all IP @@ -1628,6 +1630,11 @@ Listen 192.170.2.5:8000 +

    The optional CPU-num argument specifies the CPU to which Apache child +processes listening exclusively to this address/port should be bound. +See the SingleListen directive for +detailed information. +

    See Also: DNS Issues
    See Also: @@ -2958,6 +2965,75 @@ it is far more efficient. The server is started once, and services all subsequent connections. If you intend running Apache to serve a busy site, standalone will probably be your only option.

    + +


    +

    SingleListen directive

    + +Syntax: SingleListen on|off
    +Default: SingleListen off
    +Context: server config
    +Status: core
    +Compatibility: SingleListen is only available in Apache +1.3.7 and later. +

    +This directive controls the set of IP addresses (and ports) to which +child processes listen. When SingleListen is on, each child process +listens to a single IP address and port from the list specified using +Listen directives. Listen addresses are distributed evenly across child +processes. This is a high-performance, but not necessarily +general-purpose, way to solve the thorny accept-serialization +problem found with multiple Listen addresses. When off, the +default, each child process listens to all Listen addresses +simultaneously. +

    +This directive also enables another performance optimization for systems +with multiple CPUs and multiple network interfaces. When the device +interrupts from each network interface are bound to particular CPUs and +the Apache child processes listening for requests from those interfaces +are bound to the same CPUs, performance is increased due to +cache-warming. (Listen directives specify IP-address/CPU-number pairs.) +This optimization is particularly helpful on cache-coherent non-uniform +memory access (ccNUMA) systems such as SGI's Origin series because it +also increases memory reference locality. CPU binding is disabled if +SingleListen is off. +

    +Here is an example of this: +

    Listen 111.22.33.44 0
    +Listen 111.22.33.45 1
    +Listen 111.22.33.46 2
    +Listen 111.22.33.47 3
    +SingleListen on
    +
    +Apache child processes listening to requests from 111.22.33.44 are bound +to CPU 0, child processes listening to 111.22.33.45 are bound to CPU 1, +and so on. When this configuration is used on a four-CPU system where +interrupts from the network controller for 111.22.33.44 are bound to CPU +0, interrupts from 111.22.33.45 are bound to CPU 1, and so on, +performance is increased over a system lacking such binding. Note that +binding child processes to CPUs other than those to which the network +interrupts are bound may result in worse performance. +

    +See also the Listen directive, +the status module, and +the performance tuning guide. +

    +


    StartServers directive

    diff -Naur apache_1.3.6+01-06/htdocs/manual/mod/directives.html apache_1.3.6+01-07/htdocs/manual/mod/directives.html --- apache_1.3.6+01-06/htdocs/manual/mod/directives.html Mon Mar 22 16:17:36 1999 +++ apache_1.3.6+01-07/htdocs/manual/mod/directives.html Tue Jul 20 22:27:13 1999 @@ -210,6 +210,7 @@
  • SetEnvIf
  • SetEnvIfNoCase
  • SetHandler +
  • SingleListen
  • StartServers
  • ThreadsPerChild
  • TimeOut diff -Naur apache_1.3.6+01-06/src/include/ap_config.h apache_1.3.6+01-07/src/include/ap_config.h --- apache_1.3.6+01-06/src/include/ap_config.h Mon Jul 19 16:21:55 1999 +++ apache_1.3.6+01-07/src/include/ap_config.h Tue Jul 20 22:58:29 1999 @@ -203,6 +203,7 @@ !defined(USE_SYSVSEM_SERIALIZED_ACCEPT) #define USE_FCNTL_SERIALIZED_ACCEPT #endif +#define SINGLE_LISTEN_UNSERIALIZED_ACCEPT #define HAVE_SHMGET 1 #define USE_SHMGET_SCOREBOARD #define HAVE_MMAP 1 diff -Naur apache_1.3.6+01-06/src/include/http_conf_globals.h apache_1.3.6+01-07/src/include/http_conf_globals.h --- apache_1.3.6+01-06/src/include/http_conf_globals.h Fri Jan 1 11:04:39 1999 +++ apache_1.3.6+01-07/src/include/http_conf_globals.h Tue Jul 20 22:30:01 1999 @@ -87,6 +87,10 @@ extern int ap_dump_settings; extern API_VAR_EXPORT int ap_extended_status; +#ifdef SINGLE_LISTEN_UNSERIALIZED_ACCEPT +extern int ap_single_listen; +#endif + extern char *ap_pid_fname; extern char *ap_scoreboard_fname; extern char *ap_lock_fname; diff -Naur apache_1.3.6+01-06/src/include/httpd.h apache_1.3.6+01-07/src/include/httpd.h --- apache_1.3.6+01-06/src/include/httpd.h Tue Jul 20 17:14:58 1999 +++ apache_1.3.6+01-07/src/include/httpd.h Tue Jul 20 22:31:15 1999 @@ -918,6 +918,9 @@ int fd; int used; /* Only used during restart */ int unique; /* is local_addr unique (not wild)? */ +#ifdef SINGLE_LISTEN_UNSERIALIZED_ACCEPT + int cpu; /* if ap_single_listen, cpu to bind to */ +#endif /* more stuff here, like which protocol is bound to the port */ }; diff -Naur apache_1.3.6+01-06/src/include/scoreboard.h apache_1.3.6+01-07/src/include/scoreboard.h --- apache_1.3.6+01-06/src/include/scoreboard.h Mon Jul 19 16:22:58 1999 +++ apache_1.3.6+01-07/src/include/scoreboard.h Tue Jul 20 22:32:03 1999 @@ -159,6 +159,10 @@ char request[64]; /* We just want an idea... */ server_rec *vhostrec; /* What virtual host is being accessed? */ /* SEE ABOVE FOR SAFE USAGE! */ +#ifdef SINGLE_LISTEN_UNSERIALIZED_ACCEPT + int cpu; /* cpu to which it is bound */ + struct sockaddr_in single_addr; /* if ap_single_listen, listen addr */ +#endif } short_score; typedef struct { diff -Naur apache_1.3.6+01-06/src/main/http_config.c apache_1.3.6+01-07/src/main/http_config.c --- apache_1.3.6+01-06/src/main/http_config.c Tue Jul 20 21:53:50 1999 +++ apache_1.3.6+01-07/src/main/http_config.c Tue Jul 20 22:33:40 1999 @@ -1399,6 +1399,9 @@ ap_listeners = NULL; ap_listenbacklog = DEFAULT_LISTENBACKLOG; ap_extended_status = 0; +#ifdef SINGLE_LISTEN_UNSERIALIZED_ACCEPT + ap_single_listen = 0; +#endif /* Global virtual host hash bucket pointers. Init to null. */ ap_init_vhost_config(p); @@ -1455,6 +1458,9 @@ new->fd = -1; new->used = 0; new->unique = new->local_addr.sin_addr.s_addr != htonl(INADDR_ANY); +#ifdef SINGLE_LISTEN_UNSERIALIZED_ACCEPT + new->cpu = -1; +#endif new->next = NULL; ap_listeners = new; } diff -Naur apache_1.3.6+01-06/src/main/http_core.c apache_1.3.6+01-07/src/main/http_core.c --- apache_1.3.6+01-06/src/main/http_core.c Tue Jul 20 21:53:50 1999 +++ apache_1.3.6+01-07/src/main/http_core.c Tue Jul 20 23:29:49 1999 @@ -2382,11 +2382,13 @@ } /*ARGSUSED1*/ -static const char *set_listener(cmd_parms *cmd, void *dummy, char *ips) +static const char *set_listener(cmd_parms *cmd, void *dummy, char *ips, + char *cpustr) { listen_rec *new; char *ports; unsigned short port; + int cpu; const char *err = ap_check_cmd_context(cmd, GLOBAL_ONLY); if (err != NULL) { @@ -2407,6 +2409,14 @@ ports = ips; } + cpu = -1; + if (cpustr) { + if (ap_isdigit(*cpustr)) + cpu = atoi(cpustr); + if (cpu < 0) + return "Optional CPU id must be positive number"; + } + new=ap_pcalloc(cmd->pool, sizeof(listen_rec)); new->local_addr.sin_family = AF_INET; if (ports == ips) { /* no address */ @@ -2423,6 +2433,9 @@ new->fd = -1; new->used = 0; new->unique = new->local_addr.sin_addr.s_addr != htonl(INADDR_ANY); +#ifdef SINGLE_LISTEN_UNSERIALIZED_ACCEPT + new->cpu = cpu; +#endif new->next = ap_listeners; ap_listeners = new; return NULL; @@ -2685,6 +2698,41 @@ return NULL; } +#ifdef SINGLE_LISTEN_UNSERIALIZED_ACCEPT +static int +set_binary(const char *how, int *var) +{ + int ok; + + if (!strcasecmp(how, "on") || !strcasecmp(how, "yes") || + !strcasecmp(how, "true") || !strncasecmp(how, "enable", 6)) { + *var = 1; + ok = 1; + } else if (!strcasecmp(how, "off") || !strcasecmp(how, "no") || + !strcasecmp(how, "false") || !strncasecmp(how, "disable", 7)) { + *var = 0; + ok = 1; + } else + ok = 0; + + return ok; +} +#endif + +/*ARGSUSED1*/ +static const char *set_single_listen(cmd_parms *cmd, void *dummy, char *arg) +{ + const char *err; + + err = ap_check_cmd_context(cmd, GLOBAL_ONLY); +#ifdef SINGLE_LISTEN_UNSERIALIZED_ACCEPT + if (err == NULL && !set_binary(arg, &ap_single_listen)) + err = "Unknown SingleListen state; try \"on\" or \"off\""; +#endif + + return err; +} + #ifdef WIN32 static const char *set_interpreter_source(cmd_parms *cmd, core_dir_config *d, char *arg) @@ -2882,8 +2930,8 @@ OR_ALL, TAKE12, "soft/hard limits for max number of processes per uid" }, { "BindAddress", set_bind_address, NULL, RSRC_CONF, TAKE1, "'*', a numeric IP address, or the name of a host with a unique IP address"}, -{ "Listen", set_listener, NULL, RSRC_CONF, TAKE1, - "A port number or a numeric IP address and a port number"}, +{ "Listen", set_listener, NULL, RSRC_CONF, TAKE12, + "A port number or a numeric IP address and a port number, and an optional CPU id"}, { "SendBufferSize", set_send_buffer_size, NULL, RSRC_CONF, TAKE1, "Send buffer size in bytes"}, { "AddModule", add_module_command, NULL, RSRC_CONF, ITERATE, @@ -2925,6 +2973,13 @@ (void*)XtOffsetOf(core_dir_config, limit_req_body), OR_ALL, TAKE1, "Limit (in bytes) on maximum size of request message body" }, +{ "SingleListen", set_single_listen, NULL, RSRC_CONF, TAKE1, +#ifdef SINGLE_LISTEN_UNSERIALIZED_ACCEPT + "Force each child to serve a single Listen address" +#else + "Ignored because !SINGLE_LISTEN_UNSERIALIZED_ACCEPT" +#endif +}, { NULL } }; diff -Naur apache_1.3.6+01-06/src/main/http_main.c apache_1.3.6+01-07/src/main/http_main.c --- apache_1.3.6+01-06/src/main/http_main.c Tue Jul 20 17:17:27 1999 +++ apache_1.3.6+01-07/src/main/http_main.c Tue Jul 20 22:40:59 1999 @@ -249,6 +249,9 @@ int ap_listenbacklog; int ap_dump_settings = 0; API_VAR_EXPORT int ap_extended_status = 0; +#ifdef SINGLE_LISTEN_UNSERIALIZED_ACCEPT +int ap_single_listen; +#endif /* * The max child slot ever assigned, preserved across restarts. Necessary @@ -965,7 +968,7 @@ * when it's safe in the single Listen case. */ #ifdef SINGLE_LISTEN_UNSERIALIZED_ACCEPT -#define SAFE_ACCEPT(stmt) {if(ap_listeners->next != ap_listeners) {stmt;}} +#define SAFE_ACCEPT(stmt) {if(!ap_single_listen && ap_listeners->next != ap_listeners) {stmt;}} #else #define SAFE_ACCEPT(stmt) {stmt;} #endif @@ -3376,7 +3379,11 @@ /* warn them about the starvation problem if they're using multiple * sockets */ - if (ap_listeners->next != ap_listeners) { + if ( +#ifdef SINGLE_LISTEN_UNSERIALIZED_ACCEPT + !ap_single_listen && +#endif + ap_listeners->next != ap_listeners) { ap_log_error(APLOG_MARK, APLOG_NOERRNO|APLOG_CRIT, NULL, "You cannot use multiple Listens safely on your system, " "proceeding anyway. See src/PORTING, search for " @@ -3695,6 +3702,19 @@ } #endif +#ifdef SINGLE_LISTEN_UNSERIALIZED_ACCEPT + ap_sync_scoreboard_image(); + ap_scoreboard_image->servers[my_child_num].cpu = -1; + if (ap_single_listen) { + if (head_listener->cpu >= 0 && ap_os_bind_cpu(head_listener->cpu)) + ap_scoreboard_image->servers[my_child_num].cpu = + head_listener->cpu; + ap_scoreboard_image->servers[my_child_num].single_addr = + head_listener->local_addr; + } + put_scoreboard_info(my_child_num, &ap_scoreboard_image->servers[my_child_num]); +#endif + ap_child_init_modules(pchild, server_conf); /* done with the initialization critical section */ @@ -3767,7 +3787,11 @@ SAFE_ACCEPT(accept_mutex_on()); for (;;) { - if (ap_listeners->next != ap_listeners) { + if ( +#ifdef SINGLE_LISTEN_UNSERIALIZED_ACCEPT + !ap_single_listen && +#endif + ap_listeners->next != ap_listeners) { /* more than one socket */ memcpy(&main_fds, &listenfds, sizeof(fd_set)); srv = ap_select(listenmaxfd + 1, &main_fds, NULL, NULL, NULL); @@ -4072,6 +4096,9 @@ signal(SIGINT, just_die); signal(SIGQUIT, SIG_DFL); signal(SIGTERM, just_die); +#ifdef SINGLE_LISTEN_UNSERIALIZED_ACCEPT + ap_single_listen = 0; +#endif child_main(slot); } diff -Naur apache_1.3.6+01-06/src/modules/standard/mod_status.c apache_1.3.6+01-07/src/modules/standard/mod_status.c --- apache_1.3.6+01-06/src/modules/standard/mod_status.c Thu Jul 8 12:51:33 1999 +++ apache_1.3.6+01-07/src/modules/standard/mod_status.c Tue Jul 20 22:44:40 1999 @@ -254,6 +254,10 @@ parent_score ps_record; char stat_buffer[HARD_SERVER_LIMIT]; int pid_buffer[HARD_SERVER_LIMIT]; +#ifdef SINGLE_LISTEN_UNSERIALIZED_ACCEPT + int cpu_buffer[HARD_SERVER_LIMIT]; + struct sockaddr_in addr_buffer[HARD_SERVER_LIMIT]; +#endif clock_t tu, ts, tcu, tcs; server_rec *vhost; @@ -313,6 +317,11 @@ res = score_record.status; stat_buffer[i] = status_flags[res]; pid_buffer[i] = (int) ps_record.pid; +#ifdef SINGLE_LISTEN_UNSERIALIZED_ACCEPT + cpu_buffer[i] = score_record.cpu; + if (ap_single_listen) + addr_buffer[i] = score_record.single_addr; +#endif if (res == SERVER_READY) ready++; else if (res != SERVER_DEAD) @@ -461,6 +470,26 @@ ap_rputs("
    \n", r);
     	    for (i = 0; i < HARD_SERVER_LIMIT; ++i) {
     		if (stat_buffer[i] != '.') {
    +#ifdef SINGLE_LISTEN_UNSERIALIZED_ACCEPT
    +		    char slbuf[32];
    +
    +		    if (cpu_buffer[i] >= 0)
    +			ap_snprintf(slbuf, sizeof slbuf, "cpu %d", cpu_buffer[i]);
    +		    else
    +			strcpy(slbuf, "any cpu");
    +		    ap_rprintf(r, "   %d in state: %c on %s", pid_buffer[i],
    +		     stat_buffer[i], slbuf);
    +		    if (ap_single_listen)
    +			ap_rprintf(r, " listening to %s:%hu\n",
    +			    (addr_buffer[i].sin_addr.s_addr == htonl(INADDR_ANY)) ?
    +			    "*" : ap_inet_ntoa(addr_buffer[i].sin_addr, slbuf, NULL),
    +			    ntohs(addr_buffer[i].sin_port));
    +		    else if (++j >= 2) {
    +		    	ap_rputs("\n", r);
    +			j = 0;
    +		    } else
    +		    	ap_rputs(",", r);
    +#else
     		    ap_rprintf(r, "   %d in state: %c ", pid_buffer[i],
     		     stat_buffer[i]);
     		    if (++j >= 3) {
    @@ -468,6 +497,7 @@
     			j = 0;
     		    } else
     		    	ap_rputs(",", r);
    +#endif
     		}
     	    }
     	    ap_rputs("\n", r);
    diff -Naur apache_1.3.6+01-06/src/os/unix/os.c apache_1.3.6+01-07/src/os/unix/os.c
    --- apache_1.3.6+01-06/src/os/unix/os.c	Tue Sep 15 23:49:44 1998
    +++ apache_1.3.6+01-07/src/os/unix/os.c	Tue Jul 20 22:45:13 1999
    @@ -6,6 +6,11 @@
     #include "ap_config.h"
     #include "os.h"
     
    +#if defined(IRIX) && IRIX >= 50
    +#include "httpd.h"
    +#include "http_log.h"
    +#include 
    +#endif
     
     /* some linkers complain unless there's at least one function in each
      * .o file... and extra prototype is for gcc -Wmissing-prototypes
    @@ -168,4 +173,25 @@
     #else
         return dlerror();
     #endif
    +}
    +
    +/*
    + * Bind the current process to the given CPU.  Return nonzero if
    + * successful.
    + */
    +int
    +ap_os_bind_cpu(int cpu)
    +{
    +    int bound = 0;
    +
    +#if defined(IRIX) && IRIX >= 50
    +    /* only Irix 5.0 and beyond have sysmp(MP_MUSTRUN) */
    +    if (sysmp(MP_MUSTRUN, cpu) != -1)
    +	bound = 1;
    +    else
    +	ap_log_error(APLOG_MARK, APLOG_WARNING, NULL,
    +	    "cannot bind process %d to cpu %d", getpid(), cpu);
    +#endif
    +
    +    return bound;
     }
    diff -Naur apache_1.3.6+01-06/src/os/unix/os.h apache_1.3.6+01-07/src/os/unix/os.h
    --- apache_1.3.6+01-06/src/os/unix/os.h	Thu Jul  8 12:53:52 1999
    +++ apache_1.3.6+01-07/src/os/unix/os.h	Tue Jul 20 22:45:33 1999
    @@ -140,4 +140,6 @@
     void *      ap_os_dso_sym(void *, const char *);
     const char *ap_os_dso_error(void);
     
    +int	    ap_os_bind_cpu(int);
    +
     #endif	/* !APACHE_OS_H */
    diff -Naur apache_1.3.6+01-06/src/os/win32/os.h apache_1.3.6+01-07/src/os/win32/os.h
    --- apache_1.3.6+01-06/src/os/win32/os.h	Sun Mar  7 05:13:56 1999
    +++ apache_1.3.6+01-07/src/os/win32/os.h	Tue Jul 20 22:45:58 1999
    @@ -120,4 +120,6 @@
     #define ap_os_dso_sym(h,s)  GetProcAddress(h,s)
     #define ap_os_dso_error()   ""	/* for now */
     
    +#define	ap_os_bind_cpu(c)   (0)
    +
     #endif   /* ! APACHE_OS_H */