diff -Naur apache_1.3.6+01-06/conf/httpd.conf-dist apache_1.3.6+01-07/conf/httpd.conf-dist
--- apache_1.3.6+01-06/conf/httpd.conf-dist Wed Mar 10 01:14:27 1999
+++ apache_1.3.6+01-07/conf/httpd.conf-dist Tue Jul 20 22:19:58 1999
@@ -179,6 +179,14 @@
#BindAddress *
#
+# SingleListen: Require that each child process listen to a single IP
+# address and port, from the list specified using Listen options.
+# Listen addresses are distributed evenly across child processes. This
+# avoids the whole serialized accept problem.
+#
+SingleListen off
+
+#
# Dynamic Shared Object (DSO) Support
#
# To be able to use the functionality of a module which was built as a DSO you
diff -Naur apache_1.3.6+01-06/htdocs/manual/bind.html apache_1.3.6+01-07/htdocs/manual/bind.html
--- apache_1.3.6+01-06/htdocs/manual/bind.html Mon Mar 22 16:17:33 1999
+++ apache_1.3.6+01-07/htdocs/manual/bind.html Tue Jul 20 22:21:25 1999
@@ -72,7 +72,7 @@
Syntax: Listen [ port | IP-address:port ]
+>Syntax: Listen [ port | IP-address:port ] [ CPU-num ]
Port
directive. If an IP address is given as well as a port, the server
-will listen on the given port and interface. Multiple Listen
+will listen on the given port and interface.
+
+
The optional CPU-num argument specifies a CPU to which child
+processes listening to this address should be bound, when the SingleListen option is
+on.
+
+
Multiple Listen
directives may be used to specify a number of addresses and ports to
listen to. The server will respond to requests from any of the listed
addresses and ports.
@@ -131,6 +138,7 @@
Virtual Hosts,
BindAddress directive,
Port directive,
+SingleListen directive,
DNS Issues
and
<VirtualHost> section.
diff -Naur apache_1.3.6+01-06/htdocs/manual/misc/perf-tuning.html apache_1.3.6+01-07/htdocs/manual/misc/perf-tuning.html
--- apache_1.3.6+01-06/htdocs/manual/misc/perf-tuning.html Mon Mar 22 16:17:45 1999
+++ apache_1.3.6+01-07/htdocs/manual/misc/perf-tuning.html Tue Jul 20 22:24:35 1999
@@ -238,7 +238,7 @@
contains timing indications. For highest performance, set
ExtendedStatus off (which is the default).
-
accept Serialization - multiple sockets
+accept Serialization - multiple sockets
This discusses a shortcoming in the Unix socket API.
Suppose your
diff -Naur apache_1.3.6+01-06/htdocs/manual/mod/core.html apache_1.3.6+01-07/htdocs/manual/mod/core.html
--- apache_1.3.6+01-06/htdocs/manual/mod/core.html Thu Jul 8 12:05:03 1999
+++ apache_1.3.6+01-07/htdocs/manual/mod/core.html Tue Jul 20 22:26:34 1999
@@ -92,6 +92,7 @@
ServerSignature
ServerTokens
ServerType
+SingleListen
StartServers
ThreadsPerChild
TimeOut
@@ -1575,7 +1576,7 @@
HREF="directive-dict.html#Syntax"
REL="Help"
>Syntax:
-Listen [IP address:]port number
+Listen [IP address:]port number [CPU-num]
Compatibility: Listen is only available in Apache
-1.1 and later.
+1.1 and later.
+The optional CPU-num argument is only available in Apache 1.3.7 and later.
The Listen directive instructs Apache to listen to more than one IP
address or port; by default it responds to requests on all IP
@@ -1628,6 +1630,11 @@
Listen 192.170.2.5:8000
+
The optional CPU-num argument specifies the CPU to which Apache child
+processes listening exclusively to this address/port should be bound.
+See the SingleListen directive for
+detailed information.
+
See Also:
DNS Issues
See Also:
@@ -2958,6 +2965,75 @@
it is far more efficient. The server is started once, and services all
subsequent connections. If you intend running Apache to serve a busy site,
standalone will probably be your only option.
+
+
+
+
+Syntax: SingleListen on|off
+Default: SingleListen off
+Context: server config
+Status: core
+Compatibility: SingleListen is only available in Apache
+1.3.7 and later.
+
+This directive controls the set of IP addresses (and ports) to which
+child processes listen. When SingleListen is on, each child process
+listens to a single IP address and port from the list specified using
+Listen directives. Listen addresses are distributed evenly across child
+processes. This is a high-performance, but not necessarily
+general-purpose, way to solve the thorny accept-serialization
+problem found with multiple Listen addresses. When off, the
+default, each child process listens to all Listen addresses
+simultaneously.
+
+This directive also enables another performance optimization for systems
+with multiple CPUs and multiple network interfaces. When the device
+interrupts from each network interface are bound to particular CPUs and
+the Apache child processes listening for requests from those interfaces
+are bound to the same CPUs, performance is increased due to
+cache-warming. (Listen directives specify IP-address/CPU-number pairs.)
+This optimization is particularly helpful on cache-coherent non-uniform
+memory access (ccNUMA) systems such as SGI's Origin series because it
+also increases memory reference locality. CPU binding is disabled if
+SingleListen is off.
+
+Here is an example of this:
+
Listen 111.22.33.44 0
+Listen 111.22.33.45 1
+Listen 111.22.33.46 2
+Listen 111.22.33.47 3
+SingleListen on
+
+Apache child processes listening to requests from 111.22.33.44 are bound
+to CPU 0, child processes listening to 111.22.33.45 are bound to CPU 1,
+and so on. When this configuration is used on a four-CPU system where
+interrupts from the network controller for 111.22.33.44 are bound to CPU
+0, interrupts from 111.22.33.45 are bound to CPU 1, and so on,
+performance is increased over a system lacking such binding. Note that
+binding child processes to CPUs other than those to which the network
+interrupts are bound may result in worse performance.
+
+See also the Listen directive,
+the status module, and
+the performance tuning guide.
+
+
diff -Naur apache_1.3.6+01-06/htdocs/manual/mod/directives.html apache_1.3.6+01-07/htdocs/manual/mod/directives.html
--- apache_1.3.6+01-06/htdocs/manual/mod/directives.html Mon Mar 22 16:17:36 1999
+++ apache_1.3.6+01-07/htdocs/manual/mod/directives.html Tue Jul 20 22:27:13 1999
@@ -210,6 +210,7 @@
SetEnvIf
SetEnvIfNoCase
SetHandler
+SingleListen
StartServers
ThreadsPerChild
TimeOut
diff -Naur apache_1.3.6+01-06/src/include/ap_config.h apache_1.3.6+01-07/src/include/ap_config.h
--- apache_1.3.6+01-06/src/include/ap_config.h Mon Jul 19 16:21:55 1999
+++ apache_1.3.6+01-07/src/include/ap_config.h Tue Jul 20 22:58:29 1999
@@ -203,6 +203,7 @@
!defined(USE_SYSVSEM_SERIALIZED_ACCEPT)
#define USE_FCNTL_SERIALIZED_ACCEPT
#endif
+#define SINGLE_LISTEN_UNSERIALIZED_ACCEPT
#define HAVE_SHMGET 1
#define USE_SHMGET_SCOREBOARD
#define HAVE_MMAP 1
diff -Naur apache_1.3.6+01-06/src/include/http_conf_globals.h apache_1.3.6+01-07/src/include/http_conf_globals.h
--- apache_1.3.6+01-06/src/include/http_conf_globals.h Fri Jan 1 11:04:39 1999
+++ apache_1.3.6+01-07/src/include/http_conf_globals.h Tue Jul 20 22:30:01 1999
@@ -87,6 +87,10 @@
extern int ap_dump_settings;
extern API_VAR_EXPORT int ap_extended_status;
+#ifdef SINGLE_LISTEN_UNSERIALIZED_ACCEPT
+extern int ap_single_listen;
+#endif
+
extern char *ap_pid_fname;
extern char *ap_scoreboard_fname;
extern char *ap_lock_fname;
diff -Naur apache_1.3.6+01-06/src/include/httpd.h apache_1.3.6+01-07/src/include/httpd.h
--- apache_1.3.6+01-06/src/include/httpd.h Tue Jul 20 17:14:58 1999
+++ apache_1.3.6+01-07/src/include/httpd.h Tue Jul 20 22:31:15 1999
@@ -918,6 +918,9 @@
int fd;
int used; /* Only used during restart */
int unique; /* is local_addr unique (not wild)? */
+#ifdef SINGLE_LISTEN_UNSERIALIZED_ACCEPT
+ int cpu; /* if ap_single_listen, cpu to bind to */
+#endif
/* more stuff here, like which protocol is bound to the port */
};
diff -Naur apache_1.3.6+01-06/src/include/scoreboard.h apache_1.3.6+01-07/src/include/scoreboard.h
--- apache_1.3.6+01-06/src/include/scoreboard.h Mon Jul 19 16:22:58 1999
+++ apache_1.3.6+01-07/src/include/scoreboard.h Tue Jul 20 22:32:03 1999
@@ -159,6 +159,10 @@
char request[64]; /* We just want an idea... */
server_rec *vhostrec; /* What virtual host is being accessed? */
/* SEE ABOVE FOR SAFE USAGE! */
+#ifdef SINGLE_LISTEN_UNSERIALIZED_ACCEPT
+ int cpu; /* cpu to which it is bound */
+ struct sockaddr_in single_addr; /* if ap_single_listen, listen addr */
+#endif
} short_score;
typedef struct {
diff -Naur apache_1.3.6+01-06/src/main/http_config.c apache_1.3.6+01-07/src/main/http_config.c
--- apache_1.3.6+01-06/src/main/http_config.c Tue Jul 20 21:53:50 1999
+++ apache_1.3.6+01-07/src/main/http_config.c Tue Jul 20 22:33:40 1999
@@ -1399,6 +1399,9 @@
ap_listeners = NULL;
ap_listenbacklog = DEFAULT_LISTENBACKLOG;
ap_extended_status = 0;
+#ifdef SINGLE_LISTEN_UNSERIALIZED_ACCEPT
+ ap_single_listen = 0;
+#endif
/* Global virtual host hash bucket pointers. Init to null. */
ap_init_vhost_config(p);
@@ -1455,6 +1458,9 @@
new->fd = -1;
new->used = 0;
new->unique = new->local_addr.sin_addr.s_addr != htonl(INADDR_ANY);
+#ifdef SINGLE_LISTEN_UNSERIALIZED_ACCEPT
+ new->cpu = -1;
+#endif
new->next = NULL;
ap_listeners = new;
}
diff -Naur apache_1.3.6+01-06/src/main/http_core.c apache_1.3.6+01-07/src/main/http_core.c
--- apache_1.3.6+01-06/src/main/http_core.c Tue Jul 20 21:53:50 1999
+++ apache_1.3.6+01-07/src/main/http_core.c Tue Jul 20 23:29:49 1999
@@ -2382,11 +2382,13 @@
}
/*ARGSUSED1*/
-static const char *set_listener(cmd_parms *cmd, void *dummy, char *ips)
+static const char *set_listener(cmd_parms *cmd, void *dummy, char *ips,
+ char *cpustr)
{
listen_rec *new;
char *ports;
unsigned short port;
+ int cpu;
const char *err = ap_check_cmd_context(cmd, GLOBAL_ONLY);
if (err != NULL) {
@@ -2407,6 +2409,14 @@
ports = ips;
}
+ cpu = -1;
+ if (cpustr) {
+ if (ap_isdigit(*cpustr))
+ cpu = atoi(cpustr);
+ if (cpu < 0)
+ return "Optional CPU id must be positive number";
+ }
+
new=ap_pcalloc(cmd->pool, sizeof(listen_rec));
new->local_addr.sin_family = AF_INET;
if (ports == ips) { /* no address */
@@ -2423,6 +2433,9 @@
new->fd = -1;
new->used = 0;
new->unique = new->local_addr.sin_addr.s_addr != htonl(INADDR_ANY);
+#ifdef SINGLE_LISTEN_UNSERIALIZED_ACCEPT
+ new->cpu = cpu;
+#endif
new->next = ap_listeners;
ap_listeners = new;
return NULL;
@@ -2685,6 +2698,41 @@
return NULL;
}
+#ifdef SINGLE_LISTEN_UNSERIALIZED_ACCEPT
+static int
+set_binary(const char *how, int *var)
+{
+ int ok;
+
+ if (!strcasecmp(how, "on") || !strcasecmp(how, "yes") ||
+ !strcasecmp(how, "true") || !strncasecmp(how, "enable", 6)) {
+ *var = 1;
+ ok = 1;
+ } else if (!strcasecmp(how, "off") || !strcasecmp(how, "no") ||
+ !strcasecmp(how, "false") || !strncasecmp(how, "disable", 7)) {
+ *var = 0;
+ ok = 1;
+ } else
+ ok = 0;
+
+ return ok;
+}
+#endif
+
+/*ARGSUSED1*/
+static const char *set_single_listen(cmd_parms *cmd, void *dummy, char *arg)
+{
+ const char *err;
+
+ err = ap_check_cmd_context(cmd, GLOBAL_ONLY);
+#ifdef SINGLE_LISTEN_UNSERIALIZED_ACCEPT
+ if (err == NULL && !set_binary(arg, &ap_single_listen))
+ err = "Unknown SingleListen state; try \"on\" or \"off\"";
+#endif
+
+ return err;
+}
+
#ifdef WIN32
static const char *set_interpreter_source(cmd_parms *cmd, core_dir_config *d,
char *arg)
@@ -2882,8 +2930,8 @@
OR_ALL, TAKE12, "soft/hard limits for max number of processes per uid" },
{ "BindAddress", set_bind_address, NULL, RSRC_CONF, TAKE1,
"'*', a numeric IP address, or the name of a host with a unique IP address"},
-{ "Listen", set_listener, NULL, RSRC_CONF, TAKE1,
- "A port number or a numeric IP address and a port number"},
+{ "Listen", set_listener, NULL, RSRC_CONF, TAKE12,
+ "A port number or a numeric IP address and a port number, and an optional CPU id"},
{ "SendBufferSize", set_send_buffer_size, NULL, RSRC_CONF, TAKE1,
"Send buffer size in bytes"},
{ "AddModule", add_module_command, NULL, RSRC_CONF, ITERATE,
@@ -2925,6 +2973,13 @@
(void*)XtOffsetOf(core_dir_config, limit_req_body),
OR_ALL, TAKE1,
"Limit (in bytes) on maximum size of request message body" },
+{ "SingleListen", set_single_listen, NULL, RSRC_CONF, TAKE1,
+#ifdef SINGLE_LISTEN_UNSERIALIZED_ACCEPT
+ "Force each child to serve a single Listen address"
+#else
+ "Ignored because !SINGLE_LISTEN_UNSERIALIZED_ACCEPT"
+#endif
+},
{ NULL }
};
diff -Naur apache_1.3.6+01-06/src/main/http_main.c apache_1.3.6+01-07/src/main/http_main.c
--- apache_1.3.6+01-06/src/main/http_main.c Tue Jul 20 17:17:27 1999
+++ apache_1.3.6+01-07/src/main/http_main.c Tue Jul 20 22:40:59 1999
@@ -249,6 +249,9 @@
int ap_listenbacklog;
int ap_dump_settings = 0;
API_VAR_EXPORT int ap_extended_status = 0;
+#ifdef SINGLE_LISTEN_UNSERIALIZED_ACCEPT
+int ap_single_listen;
+#endif
/*
* The max child slot ever assigned, preserved across restarts. Necessary
@@ -965,7 +968,7 @@
* when it's safe in the single Listen case.
*/
#ifdef SINGLE_LISTEN_UNSERIALIZED_ACCEPT
-#define SAFE_ACCEPT(stmt) {if(ap_listeners->next != ap_listeners) {stmt;}}
+#define SAFE_ACCEPT(stmt) {if(!ap_single_listen && ap_listeners->next != ap_listeners) {stmt;}}
#else
#define SAFE_ACCEPT(stmt) {stmt;}
#endif
@@ -3376,7 +3379,11 @@
/* warn them about the starvation problem if they're using multiple
* sockets
*/
- if (ap_listeners->next != ap_listeners) {
+ if (
+#ifdef SINGLE_LISTEN_UNSERIALIZED_ACCEPT
+ !ap_single_listen &&
+#endif
+ ap_listeners->next != ap_listeners) {
ap_log_error(APLOG_MARK, APLOG_NOERRNO|APLOG_CRIT, NULL,
"You cannot use multiple Listens safely on your system, "
"proceeding anyway. See src/PORTING, search for "
@@ -3695,6 +3702,19 @@
}
#endif
+#ifdef SINGLE_LISTEN_UNSERIALIZED_ACCEPT
+ ap_sync_scoreboard_image();
+ ap_scoreboard_image->servers[my_child_num].cpu = -1;
+ if (ap_single_listen) {
+ if (head_listener->cpu >= 0 && ap_os_bind_cpu(head_listener->cpu))
+ ap_scoreboard_image->servers[my_child_num].cpu =
+ head_listener->cpu;
+ ap_scoreboard_image->servers[my_child_num].single_addr =
+ head_listener->local_addr;
+ }
+ put_scoreboard_info(my_child_num, &ap_scoreboard_image->servers[my_child_num]);
+#endif
+
ap_child_init_modules(pchild, server_conf);
/* done with the initialization critical section */
@@ -3767,7 +3787,11 @@
SAFE_ACCEPT(accept_mutex_on());
for (;;) {
- if (ap_listeners->next != ap_listeners) {
+ if (
+#ifdef SINGLE_LISTEN_UNSERIALIZED_ACCEPT
+ !ap_single_listen &&
+#endif
+ ap_listeners->next != ap_listeners) {
/* more than one socket */
memcpy(&main_fds, &listenfds, sizeof(fd_set));
srv = ap_select(listenmaxfd + 1, &main_fds, NULL, NULL, NULL);
@@ -4072,6 +4096,9 @@
signal(SIGINT, just_die);
signal(SIGQUIT, SIG_DFL);
signal(SIGTERM, just_die);
+#ifdef SINGLE_LISTEN_UNSERIALIZED_ACCEPT
+ ap_single_listen = 0;
+#endif
child_main(slot);
}
diff -Naur apache_1.3.6+01-06/src/modules/standard/mod_status.c apache_1.3.6+01-07/src/modules/standard/mod_status.c
--- apache_1.3.6+01-06/src/modules/standard/mod_status.c Thu Jul 8 12:51:33 1999
+++ apache_1.3.6+01-07/src/modules/standard/mod_status.c Tue Jul 20 22:44:40 1999
@@ -254,6 +254,10 @@
parent_score ps_record;
char stat_buffer[HARD_SERVER_LIMIT];
int pid_buffer[HARD_SERVER_LIMIT];
+#ifdef SINGLE_LISTEN_UNSERIALIZED_ACCEPT
+ int cpu_buffer[HARD_SERVER_LIMIT];
+ struct sockaddr_in addr_buffer[HARD_SERVER_LIMIT];
+#endif
clock_t tu, ts, tcu, tcs;
server_rec *vhost;
@@ -313,6 +317,11 @@
res = score_record.status;
stat_buffer[i] = status_flags[res];
pid_buffer[i] = (int) ps_record.pid;
+#ifdef SINGLE_LISTEN_UNSERIALIZED_ACCEPT
+ cpu_buffer[i] = score_record.cpu;
+ if (ap_single_listen)
+ addr_buffer[i] = score_record.single_addr;
+#endif
if (res == SERVER_READY)
ready++;
else if (res != SERVER_DEAD)
@@ -461,6 +470,26 @@
ap_rputs("\n", r);
for (i = 0; i < HARD_SERVER_LIMIT; ++i) {
if (stat_buffer[i] != '.') {
+#ifdef SINGLE_LISTEN_UNSERIALIZED_ACCEPT
+ char slbuf[32];
+
+ if (cpu_buffer[i] >= 0)
+ ap_snprintf(slbuf, sizeof slbuf, "cpu %d", cpu_buffer[i]);
+ else
+ strcpy(slbuf, "any cpu");
+ ap_rprintf(r, " %d in state: %c on %s", pid_buffer[i],
+ stat_buffer[i], slbuf);
+ if (ap_single_listen)
+ ap_rprintf(r, " listening to %s:%hu\n",
+ (addr_buffer[i].sin_addr.s_addr == htonl(INADDR_ANY)) ?
+ "*" : ap_inet_ntoa(addr_buffer[i].sin_addr, slbuf, NULL),
+ ntohs(addr_buffer[i].sin_port));
+ else if (++j >= 2) {
+ ap_rputs("\n", r);
+ j = 0;
+ } else
+ ap_rputs(",", r);
+#else
ap_rprintf(r, " %d in state: %c ", pid_buffer[i],
stat_buffer[i]);
if (++j >= 3) {
@@ -468,6 +497,7 @@
j = 0;
} else
ap_rputs(",", r);
+#endif
}
}
ap_rputs("\n", r);
diff -Naur apache_1.3.6+01-06/src/os/unix/os.c apache_1.3.6+01-07/src/os/unix/os.c
--- apache_1.3.6+01-06/src/os/unix/os.c Tue Sep 15 23:49:44 1998
+++ apache_1.3.6+01-07/src/os/unix/os.c Tue Jul 20 22:45:13 1999
@@ -6,6 +6,11 @@
#include "ap_config.h"
#include "os.h"
+#if defined(IRIX) && IRIX >= 50
+#include "httpd.h"
+#include "http_log.h"
+#include
+#endif
/* some linkers complain unless there's at least one function in each
* .o file... and extra prototype is for gcc -Wmissing-prototypes
@@ -168,4 +173,25 @@
#else
return dlerror();
#endif
+}
+
+/*
+ * Bind the current process to the given CPU. Return nonzero if
+ * successful.
+ */
+int
+ap_os_bind_cpu(int cpu)
+{
+ int bound = 0;
+
+#if defined(IRIX) && IRIX >= 50
+ /* only Irix 5.0 and beyond have sysmp(MP_MUSTRUN) */
+ if (sysmp(MP_MUSTRUN, cpu) != -1)
+ bound = 1;
+ else
+ ap_log_error(APLOG_MARK, APLOG_WARNING, NULL,
+ "cannot bind process %d to cpu %d", getpid(), cpu);
+#endif
+
+ return bound;
}
diff -Naur apache_1.3.6+01-06/src/os/unix/os.h apache_1.3.6+01-07/src/os/unix/os.h
--- apache_1.3.6+01-06/src/os/unix/os.h Thu Jul 8 12:53:52 1999
+++ apache_1.3.6+01-07/src/os/unix/os.h Tue Jul 20 22:45:33 1999
@@ -140,4 +140,6 @@
void * ap_os_dso_sym(void *, const char *);
const char *ap_os_dso_error(void);
+int ap_os_bind_cpu(int);
+
#endif /* !APACHE_OS_H */
diff -Naur apache_1.3.6+01-06/src/os/win32/os.h apache_1.3.6+01-07/src/os/win32/os.h
--- apache_1.3.6+01-06/src/os/win32/os.h Sun Mar 7 05:13:56 1999
+++ apache_1.3.6+01-07/src/os/win32/os.h Tue Jul 20 22:45:58 1999
@@ -120,4 +120,6 @@
#define ap_os_dso_sym(h,s) GetProcAddress(h,s)
#define ap_os_dso_error() "" /* for now */
+#define ap_os_bind_cpu(c) (0)
+
#endif /* ! APACHE_OS_H */