[Spread-users] [PATCH] Added handling of SO_REUSEADDR
Daniel Rall
dlr at finemaltcoding.com
Wed Jan 9 16:02:38 EST 2002
This patch implements David Shaw's <dshaw at akamai.com> suggestions
(http://lists.spread.org/pipermail/spread-users/2001-September/000359.html)
for handling of the SO_REUSEADDR socket option; most daemons
(including Apache's httpd) use this option. David's suggestions were
in response to Jonathan Stanton's review of the behavior of
SO_REUSEADDR across various x86 OSes
(http://lists.spread.org/pipermail/spread-users/2001-September/000354.html).
1) If the config file gives a specific interface (i.e. not
INADDR_ANY), then enable REUSEADDR. If it does not give a specific
interface (i.e. INADDR_ANY) then do not enable REUSEADDR.
2) Add a config file option to force REUSEADDR on or off which
overrides the setting from #1.
In a high-availability environment, it is necessary to have the option
of undelayed restarts. Without SO_REUSEADDR, delay times of
approximately 30 seconds on a loaded RedHat 6.2 box with 2.2.17 kernel
are exhibitted between Spread daemon stop and starts. These restart
delays are caused by the OS hanging on to the TCP listener port after
process shutdown, keeping the port open and rendering the daemon
unable to immediately re-bind to the port.
This patch was reviewed by Manoj Kasichainula <manoj at collab.net>.
Index: TODO
===================================================================
RCS file: /storage/cvsroot/spread/daemon/TODO,v
retrieving revision 1.10
diff -u -r1.10 TODO
--- TODO 31 Oct 2001 23:03:56 -0000 1.10
+++ TODO 9 Jan 2002 21:01:43 -0000
@@ -1,6 +1,5 @@
Features, ideas, and other things that might get done.
------------------------------------------------------
-* Improve REUSEADR_handling
* Allow entire class C subnet to be in config file--as long as no more then 128 are active.
* Improve stability under high load
* Add better error checks to f* functions in log.c
Index: config_gram.l
===================================================================
RCS file: /storage/cvsroot/spread/daemon/config_gram.l,v
retrieving revision 1.1.1.1
diff -u -r1.1.1.1 config_gram.l
--- config_gram.l 21 Aug 2001 14:28:21 -0000 1.1.1.1
+++ config_gram.l 9 Jan 2002 21:01:44 -0000
@@ -76,6 +76,7 @@
EventTimeStamp { return EVENTTIMESTAMP; }
DebugFlags { return DEBUGFLAGS; }
DangerousMonitor { return DANGEROUSMONITOR; }
+SocketPortReuse { return SOCKETPORTREUSE; }
RequiredAuthMethods { return REQUIREDAUTHMETHODS; }
AllowedAuthMethods { return ALLOWEDAUTHMETHODS; }
AccessControlPolicy { return ACCESSCONTROLPOLICY; }
Index: config_parse.y
===================================================================
RCS file: /storage/cvsroot/spread/daemon/config_parse.y,v
retrieving revision 1.1.1.1
diff -u -r1.1.1.1 config_parse.y
--- config_parse.y 21 Aug 2001 14:28:21 -0000 1.1.1.1
+++ config_parse.y 9 Jan 2002 21:01:45 -0000
@@ -187,7 +187,7 @@
%token DDEBUG DEXIT DPRINT DDATA_LINK DNETWORK DPROTOCOL DSESSION
%token DCONF DMEMB DFLOW_CONTROL DSTATUS DEVENTS DGROUPS DMEMORY
%token DSKIPLIST DACM DALL DNONE
-%token DANGEROUSMONITOR ALLOWEDAUTHMETHODS REQUIREDAUTHMETHODS ACCESSCONTROLPOLICY
+%token DANGEROUSMONITOR SOCKETPORTREUSE ALLOWEDAUTHMETHODS REQUIREDAUTHMETHODS ACCESSCONTROLPOLICY
%token SP_BOOL LINKPROTOCOL PHOP PTCPHOP
%token IMONITOR ICLIENT IDAEMON
%token ROUTEMATRIX LINKCOST
@@ -271,6 +271,26 @@
Conf_set_dangerous_monitor_state($3.boolean);
}
}
+ | SOCKETPORTREUSE EQUALS STRING
+ {
+ port_reuse state;
+ char option[5];
+ strncpy(option, $3.string, 5);
+ if (strcasecmp(option, "on") == 0)
+ {
+ state = port_reuse_on;
+ }
+ else if (strcasecmp(option, "off") == 0)
+ {
+ state = port_reuse_off;
+ }
+ else
+ {
+ /* Default to AUTO. */
+ state = port_reuse_auto;
+ }
+ Conf_set_port_reuse_type(state);
+ }
| ALLOWEDAUTHMETHODS EQUALS STRING
{
char auth_list[MAX_AUTH_LIST_LEN];
Index: configuration.c
===================================================================
RCS file: /storage/cvsroot/spread/daemon/configuration.c,v
retrieving revision 1.1.1.1
diff -u -r1.1.1.1 configuration.c
--- configuration.c 21 Aug 2001 14:28:21 -0000 1.1.1.1
+++ configuration.c 9 Jan 2002 21:01:48 -0000
@@ -84,8 +84,10 @@
* False means to ignore requests for those actions. THIS IS THE SAFE SETTING
*/
static bool EnableDangerousMonitor = FALSE;
-static int Link_Protocol;
+static port_reuse SocketPortReuse = port_reuse_auto;
+
+static int Link_Protocol;
int Conf_init( char *file_name, char *my_name )
{
@@ -517,4 +519,29 @@
return;
}
EnableDangerousMonitor = new_state;
+}
+
+port_reuse Conf_get_port_reuse_type(void)
+{
+ return(SocketPortReuse);
+}
+
+void Conf_set_port_reuse_type(port_reuse state)
+{
+ switch (state)
+ {
+ case port_reuse_auto:
+ Alarm(PRINT, "Setting SO_REUSEADDR to auto\n");
+ break;
+ case port_reuse_on:
+ Alarm(PRINT, "Setting SO_REUSEADDR to always on -- make sure Spread daemon host is secured!\n");
+ break;
+ case port_reuse_off:
+ Alarm(PRINT, "Setting SO_REUSEADDR to always off\n");
+ break;
+ default:
+ /* Inavlid type -- ignored */
+ return;
+ }
+ SocketPortReuse = state;
}
Index: configuration.h
===================================================================
RCS file: /storage/cvsroot/spread/daemon/configuration.h,v
retrieving revision 1.1.1.1
diff -u -r1.1.1.1 configuration.h
--- configuration.h 21 Aug 2001 14:28:21 -0000 1.1.1.1
+++ configuration.h 9 Jan 2002 21:01:48 -0000
@@ -77,6 +77,12 @@
segment segments[MAX_SEGMENTS];
} configuration;
+typedef enum dummy_port_reuse {
+ port_reuse_auto,
+ port_reuse_on,
+ port_reuse_off
+} port_reuse;
+
int Conf_init( char *file_name, char *my_name );
configuration Conf(void);
proc Conf_my(void);
@@ -96,6 +102,8 @@
bool Conf_get_dangerous_monitor_state(void);
void Conf_set_dangerous_monitor_state(bool new_state);
+port_reuse Conf_get_port_reuse_type(void);
+void Conf_set_port_reuse_type(port_reuse state);
int Conf_get_link_protocol(void);
void Conf_set_link_protocol(int protocol);
Index: sample.spread.conf
===================================================================
RCS file: /storage/cvsroot/spread/daemon/sample.spread.conf,v
retrieving revision 1.2
diff -u -r1.2 sample.spread.conf
--- sample.spread.conf 31 Aug 2001 03:03:59 -0000 1.2
+++ sample.spread.conf 9 Jan 2002 21:01:49 -0000
@@ -74,6 +74,18 @@
#DangerousMonitor = false
+#Set handling of SO_REUSEADDR socket option for the daemon's TCP
+# listener. This is useful for facilitating quick daemon restarts (OSes
+# often hold onto the interface/port combination for a short period of time
+# after daemon shut down).
+#
+# AUTO - Active when bound to specific interfaces (default).
+# ON - Always active, regardless of interface.
+# SECURITY RISK FOR ANY OS WHICH ALLOW DOUBLE BINDS BY DIFFERENT USERS
+# OFF - Always off.
+
+#SocketPortReuse = AUTO
+
#Set the list of authentication methods that the daemon will allow
# and those which are required in all cases.
# All of the methods listed in "RequiredAuthMethods" will be checked,
Index: session.c
===================================================================
RCS file: /storage/cvsroot/spread/daemon/session.c,v
retrieving revision 1.4
diff -u -r1.4 session.c
--- session.c 31 Oct 2001 23:06:48 -0000 1.4
+++ session.c 9 Jan 2002 21:01:55 -0000
@@ -112,6 +112,7 @@
static void Sess_deliver_reject( message_obj *msg );
static void Sess_create_reject_message ( message_obj *msg );
static int Sess_get_p2p_dests( int num_groups, char groups[][MAX_GROUP_NAME], char dests[][MAX_GROUP_NAME] );
+static void Sess_activate_port_reuse( mailbox mbox );
int count_bits_set( int32u field, int first_index, int last_index)
{
@@ -184,13 +185,20 @@
{
if (Is_IfType_Client(My.ifc[i].type) || Is_IfType_Any(My.ifc[i].type) )
{
+ port_reuse type = Conf_get_port_reuse_type();
if( (mbox = socket( AF_INET, SOCK_STREAM, 0 ) ) == -1)
Alarm( EXIT, "Sess_init: INET sock error\n" );
+ if (type == port_reuse_on)
+ Sess_activate_port_reuse(mbox);
if (Is_IfType_Any(My.ifc[i].type) )
inet_addr.sin_addr.s_addr = INADDR_ANY;
else
+ {
+ if (type == port_reuse_auto)
+ Sess_activate_port_reuse(mbox);
inet_addr.sin_addr.s_addr = htonl(My.ifc[i].ip);
+ }
if( bind( mbox, (struct sockaddr *)&inet_addr, sizeof(inet_addr) ) == -1)
{
Alarm( PRINT, "Sess_init: INET unable to bind to port %d, already running \n" ,port );
@@ -1812,4 +1820,11 @@
head_ptr->type = Flip_int32( head_ptr->type );
head_ptr->num_groups = Flip_int32( head_ptr->num_groups );
head_ptr->data_len = Flip_int32( head_ptr->data_len );
+}
+
+void Sess_activate_port_reuse( mailbox mbox )
+{
+ int on = 1;
+ if (setsockopt(mbox, SOL_SOCKET, SO_REUSEADDR, (void *) &on, sizeof(on)) < 0)
+ Alarm( EXIT, "Sess_init: SO_REUSEADDR socket option error\n" );
}
More information about the Spread-users
mailing list