[Spread-cvs] commit: r512 - branches/reconfig_groups_crash_bugfix/daemon
jonathan at spread.org
jonathan at spread.org
Thu Apr 19 23:27:46 EDT 2012
Author: jonathan
Date: 2012-04-19 23:27:46 -0400 (Thu, 19 Apr 2012)
New Revision: 512
Modified:
branches/reconfig_groups_crash_bugfix/daemon/configuration.c
branches/reconfig_groups_crash_bugfix/daemon/configuration.h
branches/reconfig_groups_crash_bugfix/daemon/groups.c
branches/reconfig_groups_crash_bugfix/daemon/groups.h
branches/reconfig_groups_crash_bugfix/daemon/protocol.c
branches/reconfig_groups_crash_bugfix/daemon/session.c
Log:
Complete initial implementation of bugfix. On conf reload the G_signal_conf_reload() function is called and it rebuilds the groups DaemonList structures to be indexed based on the new conf structure. Also export the Conf_config_copy() function for use outside configuration.c. Add new verisons of G_compare_proc_ids_by_conf() functions for different active configurations.
Modified: branches/reconfig_groups_crash_bugfix/daemon/configuration.c
===================================================================
--- branches/reconfig_groups_crash_bugfix/daemon/configuration.c 2012-04-19 17:03:22 UTC (rev 511)
+++ branches/reconfig_groups_crash_bugfix/daemon/configuration.c 2012-04-20 03:27:46 UTC (rev 512)
@@ -114,8 +114,6 @@
static char Conf_MyName_buf[80];
static char *Conf_MyName;
-static void Conf_config_copy( configuration *src_conf, configuration *dst_conf);
-
/* Hash function for string to 32 bit int */
static LOC_INLINE int32u conf_hash_string(const void * key, int32u key_len)
{
@@ -559,7 +557,7 @@
return( -1 );
}
-static void Conf_config_copy( configuration *src_conf, configuration *dst_conf)
+void Conf_config_copy( configuration *src_conf, configuration *dst_conf)
{
int i,j,p_index;
proc p;
Modified: branches/reconfig_groups_crash_bugfix/daemon/configuration.h
===================================================================
--- branches/reconfig_groups_crash_bugfix/daemon/configuration.h 2012-04-19 17:03:22 UTC (rev 511)
+++ branches/reconfig_groups_crash_bugfix/daemon/configuration.h 2012-04-20 03:27:46 UTC (rev 512)
@@ -93,6 +93,7 @@
void Conf_init( char *file_name, char *my_name );
void Conf_load_conf_file( char *file_name, char *my_name );
+void Conf_config_copy( configuration *src_conf, configuration *dst_conf);
configuration Conf(void);
configuration *Conf_ref(void);
proc Conf_my(void);
Modified: branches/reconfig_groups_crash_bugfix/daemon/groups.c
===================================================================
--- branches/reconfig_groups_crash_bugfix/daemon/groups.c 2012-04-19 17:03:22 UTC (rev 511)
+++ branches/reconfig_groups_crash_bugfix/daemon/groups.c 2012-04-20 03:27:46 UTC (rev 512)
@@ -107,6 +107,8 @@
return grp->name;
}
+static configuration Cn_active;
+static configuration Cn_interim;
static int Gstate;
static configuration Trans_memb;
static membership_id Trans_memb_id;
@@ -174,6 +176,7 @@
static int G_compare_cstrptr(const void *, const void *);
static int G_compare_proc_ids_by_conf( const void *, const void * );
+static int G_compare_proc_ids_by_conf_internal( configuration *config, const void *a, const void *b);
static int G_compare_daemon_vs_set( const void *, const void * );
static int G_compare_memb_ids( const membership_id *mid1,
@@ -188,14 +191,52 @@
static int G_compare_proc_ids_by_conf(const void *a, const void *b)
{
+ return G_compare_proc_ids_by_conf_internal( &Cn_active, a, b);
+}
+
+static int G_compare_proc_ids_by_conf_interim(const void *a, const void *b)
+{
+ return G_compare_proc_ids_by_conf_internal( &Cn_interim, a, b);
+}
+
+/* Compare two procs based on the specified configuration structure
+ * The ordering is defined by the order the procs occur in the configuration.
+ * If a proc is no longer in the active configuration, it is ordered after all active procs
+ * based on it's IP address.
+ */
+static int G_compare_proc_ids_by_conf_internal( configuration *config, const void *a, const void *b)
+{
proc dummy_proc;
- int ia = Conf_proc_by_id( **(const int32**) a, &dummy_proc );
- int ib = Conf_proc_by_id( **(const int32**) b, &dummy_proc );
+ const int32u aip = **(const int32u**)a;
+ const int32u bip = **(const int32u**)b;
- assert( ia > -1 );
- assert( ib > -1 );
+ int ia = Conf_proc_by_id_in_conf( config, aip, &dummy_proc );
+ int ib = Conf_proc_by_id_in_conf( config, bip, &dummy_proc );
- return ia - ib;
+ /* common case */
+ if (ia > -1 && ib > -1) {
+ /* both are active */
+ return ia - ib;
+ }
+ if (ia < 0 && ib < 0 ) {
+ /* both not active so compare IP addresses */
+ if (aip == bip)
+ return 0;
+ else
+ if ( aip < bip )
+ return -1;
+ else
+ return 1;
+
+ } else
+ if (ia < 0)
+ return 1;
+ else
+ if (ib < 0)
+ return -1;
+ else
+ /* dup rule for all are active to keep compiler happy */
+ return ia - ib;
}
static int G_compare_daemon_vs_set(const void *a, const void *b)
@@ -255,6 +296,11 @@
MySyncedSet.proc_ids[0] = My.id;
G_print_synced_set( SPLOG_INFO, &MySyncedSet, "G_init" );
+ Cn_active.allprocs = Mem_alloc( MAX_PROCS_RING * sizeof( proc ) );
+ if (Cn_active.allprocs == NULL) {
+ Alarmp( SPLOG_FATAL, GROUPS, "G_init: Failed to allocate memory for Cn_active procs array\n");
+ }
+
ret = stdskl_construct(&GroupsList, sizeof(group*), 0, G_compare_cstrptr);
if (ret != 0) {
Alarmp( SPLOG_FATAL, GROUPS, "G_init: Failure to Initialize GroupsList\n");
@@ -298,9 +344,180 @@
Gathered.complete = 0;
Gathered.next = NULL;
+ Conf_config_copy( Conf_ref(), &Cn_active );
+
G_shift_to_GOP();
}
+
+/*
+ * Called from Prot_initiate_conf_reload after configuration file is reloaded (potentially with changes to spread configuration)
+ * Needs to update any static-scope variables that depend on current configuration
+ *
+ Algorithm to clean up existing DaemonList skiplists that are stored for every group.
+ Each skiplist needs to be reformed with a different comparison function using the new Config structure.
+ Once they are all reformed, we need to move the temporary daemon lists back into the main GroupsList structure entries.
+
+ Steps:
+ 1)
+ make new grp skiplist to store temporary copies of all of the daemon lists.
+ loop over all grps:
+ copy damon list from current skiplist to new skiplist with comparison function (G_compare_proc_ids_by_conf_interim) that uses the new config structure. This maintains the daemon_members in each entry, but inserts them into a new skiplist in a new order based on the new configuration.
+ store new skiplist in the new grp list so we can store all of them with new structures before destroying them all and switching the active Conf
+ end
+ 2)
+ loop over all grps:
+ destruct the current daemonlist for all groups
+ end
+ 3)
+ Set Config variable to point to new config (old config no longer available)
+ 4)
+ loop over all grps:
+ create new daemon skiplsit for each group with normal comparison function (will effectively use new config now)
+ get begin and end iterators for temp daemon skiplist with stdstk_begin() and stdskl_end()
+ call stdstk_insert_seq() to insert contents of temp skiplist into new permantent skiplist
+ destrcut temp skiplist
+ end
+ destruct temp grp skiplist.
+
+ At the end of this function (which runs atomically with regards to any other groups functions):
+ - all of the skiplists with daemons in them will be valid with the new Conf structure and can find the 'old' daemons.
+ - There are no changes to the handling of the groups membership code that runs when the ring is reformed as it can correctly remove the daemons when it thinks it should beauase they are correctly indexed and sorted in the skiplist.
+
+ */
+
+void G_signal_conf_reload(void)
+{
+ int ret;
+ group *grp, *tmp_grp;
+ daemon_members *dmn;
+ stdit git, dit;
+ stdskl tmp_GroupsList;
+
+ ret = stdskl_construct(&tmp_GroupsList, sizeof(group*), 0, G_compare_cstrptr);
+ if (ret != 0) {
+ Alarmp( SPLOG_FATAL, GROUPS, "G_init: Failure to Initialize GroupsList\n");
+ }
+
+ /*
+ 0) Get copy of new configuration to use while we still have old config also.
+ A shallow copy is fine as:
+ - we want the current Conf(),
+ - do not need our own copy of allprocs,
+ - and only need it valid for this function call.
+ */
+
+ Cn_interim = Conf();
+
+ /*
+ 1)
+ loop over all grps:
+ copy damon list from current skiplist to new skiplist with new config as comparison fucntion
+ store new skiplist in the new grp list.
+ end
+ */
+ for (stdskl_begin(&GroupsList, &git); !stdskl_is_end(&GroupsList, &git); )
+ {
+ grp = *(group**) stdskl_it_key(&git);
+ stdskl_it_next(&git); /* NOTE: need to do advancement before potential erasure below */
+
+ tmp_grp = new( GROUP );
+ memset( tmp_grp->name, 0, MAX_GROUP_NAME );
+ strcpy( tmp_grp->name, grp->name );
+
+ if (stdskl_construct(&tmp_grp->DaemonsList, sizeof(daemon_members*), 0, G_compare_proc_ids_by_conf_interim) != 0) {
+ Alarmp( SPLOG_FATAL, GROUPS, "%s: %d: memory allocation failed\n", __FILE__, __LINE__ );
+ }
+ tmp_grp->changed = FALSE;
+ tmp_grp->num_members = 0;
+ tmp_grp->grp_id = grp->grp_id;
+
+ if (stdskl_put(&tmp_GroupsList, NULL, &tmp_grp, NULL, STDFALSE) != 0) {
+ Alarmp( SPLOG_FATAL, GROUPS, "%s: %d: memory allocation failed\n", __FILE__, __LINE__ );
+ }
+
+ for (stdskl_begin(&grp->DaemonsList, &dit); !stdskl_is_end(&grp->DaemonsList, &dit); )
+ {
+ dmn = *(daemon_members**) stdskl_it_key(&dit);
+ stdskl_it_next(&dit); /* NOTE: need to do advancement before potential erasure below */
+
+ /* insert this dmn into the new skiplist */
+ if (stdskl_put(&tmp_grp->DaemonsList, NULL, &dmn, NULL, STDFALSE) != 0) {
+ Alarmp( SPLOG_FATAL, GROUPS, "%s: %d: memory allocation failed\n", __FILE__, __LINE__ );
+ }
+ }
+
+ }
+ /*
+ 2)
+ loop over all grps:
+ destruct the current daemonlist
+ end
+ */
+ for (stdskl_begin(&GroupsList, &git); !stdskl_is_end(&GroupsList, &git); )
+ {
+ grp = *(group**) stdskl_it_key(&git);
+ stdskl_it_next(&git); /* NOTE: need to do advancement before potential erasure below */
+
+ stdskl_destruct( &grp->DaemonsList);
+ }
+
+ /* 3)
+ Set Cn_active variable to point to new config (old config no longer available)
+ This has to be a deep copy because it must preserve all of the procs array when the configuration is next reloaded
+ and the configuration.c version of Conf changes.
+ */
+ Conf_config_copy( Conf_ref(), &Cn_active );
+
+ /* 4)
+ loop over all grps:
+ create new daemon skiplsit for each group with normal comparison function (will effectively use new config now)
+ get begin and end iterators for temp daemon skiplist with stdstk_begin() and stdskl_end()
+ call stdstk_insert_seq() to insert contents of temp skiplist into new permantent skiplist
+ destruct temp skiplist
+ free temp grp structure
+ end
+ destruct temp grp skiplist.
+ */
+
+ for (stdskl_begin(&GroupsList, &git); !stdskl_is_end(&GroupsList, &git); )
+ {
+ stdit bskl, eskl, it;
+
+ grp = *(group**) stdskl_it_key(&git);
+ stdskl_it_next(&git); /* NOTE: need to do advancement before potential erasure below */
+
+ if (stdskl_construct(&grp->DaemonsList, sizeof(daemon_members*), 0, G_compare_proc_ids_by_conf) != 0) {
+ Alarmp( SPLOG_FATAL, GROUPS, "%s: %d: memory allocation failed\n", __FILE__, __LINE__ );
+ }
+
+ /* find tmp_grp corresponding to grp */
+ stdskl_find(&tmp_GroupsList, &it, &grp->name);
+ if (stdskl_is_end(&tmp_GroupsList, &it)) {
+ Alarmp( SPLOG_FATAL, GROUPS, "G_signal_conf_reload: failed to find group (%s) in tmp_GroupsList\n", grp->name);
+ }
+ tmp_grp = *(group**) stdskl_it_key(&it);
+
+ stdskl_begin(&tmp_grp->DaemonsList, &bskl);
+ stdskl_last(&tmp_grp->DaemonsList, &eskl);
+ stdskl_begin(&grp->DaemonsList, &dit);
+ /* Insert entire tmp_grp->DaemonsList (from begin to last) into grp->DaemonsList */
+ stdskl_insert_seq(&grp->DaemonsList, &dit, &bskl, &eskl, STDTRUE);
+
+ /* destroy interim DaemonList since we are done with it */
+ stdskl_erase(&tmp_GroupsList, &it);
+ stdskl_destruct( &tmp_grp->DaemonsList);
+ dispose(tmp_grp);
+ }
+
+ if ( ! stdskl_empty( &tmp_GroupsList) ) {
+ Alarmp( SPLOG_FATAL, GROUPS, "G_signal_conf_reload: About to destroy temporary GroupsList but it isn't empty.\n");
+ }
+ stdskl_destruct( &tmp_GroupsList );
+
+}
+
+
void G_handle_reg_memb( configuration reg_memb, membership_id reg_memb_id )
{
group *grp;
Modified: branches/reconfig_groups_crash_bugfix/daemon/groups.h
===================================================================
--- branches/reconfig_groups_crash_bugfix/daemon/groups.h 2012-04-19 17:03:22 UTC (rev 511)
+++ branches/reconfig_groups_crash_bugfix/daemon/groups.h 2012-04-20 03:27:46 UTC (rev 512)
@@ -55,6 +55,7 @@
/ MAX_GROUP_NAME )
void G_init(void);
+void G_signal_conf_reload(void);
void G_handle_reg_memb( configuration reg_memb, membership_id reg_memb_id );
void G_handle_trans_memb( configuration trans_memb,
Modified: branches/reconfig_groups_crash_bugfix/daemon/protocol.c
===================================================================
--- branches/reconfig_groups_crash_bugfix/daemon/protocol.c 2012-04-19 17:03:22 UTC (rev 511)
+++ branches/reconfig_groups_crash_bugfix/daemon/protocol.c 2012-04-20 03:27:46 UTC (rev 512)
@@ -754,6 +754,7 @@
/* Signal all subsystems to update Conf and My strucures */
Net_signal_conf_reload();
Memb_signal_conf_reload();
+ /* Note: the Sess_signal also calls G_signal_conf_reload() */
Sess_signal_conf_reload();
/* update protocol varialbes with new conf */
Modified: branches/reconfig_groups_crash_bugfix/daemon/session.c
===================================================================
--- branches/reconfig_groups_crash_bugfix/daemon/session.c 2012-04-19 17:03:22 UTC (rev 511)
+++ branches/reconfig_groups_crash_bugfix/daemon/session.c 2012-04-20 03:27:46 UTC (rev 512)
@@ -441,6 +441,8 @@
void Sess_signal_conf_reload(void)
{
My = Conf_my();
+
+ G_signal_conf_reload();
}
void Sess_set_active_threshold()
More information about the Spread-cvs
mailing list