[Spread-cvs] commit: r253 - in trunk: . daemon docs docs/flush docs/flush/docs docs/flush/man flush include

jschultz at spread.org jschultz at spread.org
Sun Jul 31 06:51:41 EDT 2005


Author: jschultz
Date: 2005-07-31 06:51:40 -0400 (Sun, 31 Jul 2005)
New Revision: 253

Added:
   trunk/docs/flush/
   trunk/docs/flush/docs/
   trunk/docs/flush/docs/FL_connect.html
   trunk/docs/flush/docs/FL_disconnect.html
   trunk/docs/flush/docs/FL_error.html
   trunk/docs/flush/docs/FL_flush.html
   trunk/docs/flush/docs/FL_join.html
   trunk/docs/flush/docs/FL_leave.html
   trunk/docs/flush/docs/FL_more_msgs.html
   trunk/docs/flush/docs/FL_multicast.html
   trunk/docs/flush/docs/FL_poll.html
   trunk/docs/flush/docs/FL_receive.html
   trunk/docs/flush/docs/FL_version.html
   trunk/docs/flush/docs/access.html
   trunk/docs/flush/docs/error.html
   trunk/docs/flush/docs/flush_spread_title.gif
   trunk/docs/flush/docs/functions.html
   trunk/docs/flush/docs/index.html
   trunk/docs/flush/docs/messages.html
   trunk/docs/flush/docs/misc.html
   trunk/docs/flush/index.html
   trunk/docs/flush/man/
   trunk/docs/flush/man/FL_connect.3
   trunk/docs/flush/man/FL_disconnect.3
   trunk/docs/flush/man/FL_error.3
   trunk/docs/flush/man/FL_flush.3
   trunk/docs/flush/man/FL_join.3
   trunk/docs/flush/man/FL_leave.3
   trunk/docs/flush/man/FL_more_msgs.3
   trunk/docs/flush/man/FL_multicast.3
   trunk/docs/flush/man/FL_poll.3
   trunk/docs/flush/man/FL_receive.3
   trunk/docs/flush/man/FL_scat_multicast.3
   trunk/docs/flush/man/FL_scat_receive.3
   trunk/docs/flush/man/FL_scat_subgroupcast.3
   trunk/docs/flush/man/FL_scat_unicast.3
   trunk/docs/flush/man/FL_subgroupcast.3
   trunk/docs/flush/man/FL_unicast.3
   trunk/docs/flush/man/FL_version.3
   trunk/flush/
   trunk/flush/Makefile.in
   trunk/flush/configure.in
   trunk/flush/fl.c
   trunk/flush/fl_p.h
   trunk/flush/fl_time_memb.c
   trunk/flush/scatp.c
   trunk/flush/scatp.h
   trunk/flush/sp_time_memb.c
   trunk/flush/stats.c
   trunk/flush/stats.h
   trunk/flush/user.c
   trunk/include/fl.h
Modified:
   trunk/daemon/groups.c
Log:
Added in flush code (still needs to conform to new Spread semantics) as best I thought appropriate.  Minor comments added to groups.c.



Modified: trunk/daemon/groups.c
===================================================================
--- trunk/daemon/groups.c	2005-07-28 14:50:56 UTC (rev 252)
+++ trunk/daemon/groups.c	2005-07-31 10:51:40 UTC (rev 253)
@@ -2280,13 +2280,13 @@
 			litend_ptr = litbox_ptr + 1;
 		}
 
-		/* NOTE: this code assumes that grp->mboxes contains no duplicates */
+		/* NOTE: the following code assumes that grp->mboxes contains no duplicates */
 
 		if (num_groups == 1) {  /* no need to do extra copy work if only one group -> just use litbox 'array' directly */
 		  break;
 		}
 
-		if (num_mbox != 0) {
+		if (num_mbox != 0) {  /* mboxes contains some entries already */
 		        bigend_ptr = mboxes + num_mbox;
 
 			for (; litbox_ptr != litend_ptr; ++litbox_ptr)
@@ -2314,7 +2314,7 @@
 	  bigbox_ptr = litbox_ptr;
 	  bigend_ptr = litend_ptr;
 
-	} else {
+	} else {                /* otherwise use unique entries we've built in mboxes */
 	  bigbox_ptr = mboxes;
 	  bigend_ptr = mboxes + num_mbox;
 	}

Added: trunk/docs/flush/docs/FL_connect.html
===================================================================
--- trunk/docs/flush/docs/FL_connect.html	2005-07-28 14:50:56 UTC (rev 252)
+++ trunk/docs/flush/docs/FL_connect.html	2005-07-31 10:51:40 UTC (rev 253)
@@ -0,0 +1,166 @@
+<HTML><HEAD><TITLE>Manpage of FL_connect</TITLE>
+</HEAD>
+
+<BODY bgcolor="#FFFFFF">
+
+<!--#include virtual="/includes/header-a" -->
+
+<a href="http://www.cnds.jhu.edu/research/group/flush_spread">
+<img src="flush_spread_title.gif" alt="FLUSH SPREAD" border=0>
+</a>
+
+<!--#include virtual="/includes/header-b" -->
+
+<H1>FL_connect</H1>
+Section: User Manuals (3)<BR>Updated: Dec 2000<BR>
+<HR>
+
+<H2>NAME</H2>
+
+FL_connect - connect an application to a Spread daemon using Flush Spread semantics.
+
+<H2>SYNOPSIS</H2>
+
+<B>#include &lt;fl.h&gt;</B>
+
+<P>
+<B>int FL_connect(const char *</B><I>daemon_name</I><B>, const char *</B><I>user_name</I><B>, int </B><I>priority</I><B>, int </B><I>group_membership</I><B>, mailbox *</B><I>mbox</I><B>, char *</B><I>private_name</I><B>);</B>
+
+<H2>DESCRIPTION</H2>
+
+<B>FL_connect</B>
+
+is the initial call an application must make to establish a
+connection with a Spread daemon.  All other Flush Spread calls
+refer to a valid 
+<I>mbox </I>
+
+returned by this function.
+<P>
+The 
+<I>daemon_name</I>
+
+is the name of the Spread daemon with which to connect.  It should be a
+string in one of the following forms:
+<DL COMPACT><DT><DD>
+<DL COMPACT>
+<DT><B>4803</B>
+
+<DD>
+This will connect to the Spread daemon on the local 
+machine running on port 4803.  This form cannot be
+used to connect to a Windows95/NT machine.</p>
+
+<DT><B>4803 at localhost</B>
+
+<DD>
+This will also connect to the Spread daemon 
+on the local machine running on port 4803.
+This form can be used on Windows95/NT machines.</p>
+
+<DT><B>4803 at host.domain.edu</B> or <B>4803 at 128.220.221.99</B>
+
+<DD>
+This will connect to the machine identified by either 
+domain name or IP address at the specified port.</p>
+</DL></DL>
+
+The
+<I>user_name </I>
+
+is the name this connection would like to be known as.  It must be
+unique on the machine running the spread daemon.  The name can be an
+arbitrary length string with the same character restrictions as a
+group name (mainly it cannot contain the '#' character).
+<P>
+<I>priority</I>
+
+is a boolean (non-zero true, zero false) for whether this connection
+will be a &quot;priority&quot; connection or not. Currently this has no effect.
+<P>
+<I>group_membership</I>
+
+is a boolean for whether this connection will receive group membership
+messages or not.  Usually for Flush Spread applications this parameter
+should be true.  If your application doesn't need group membership
+messages, then Spread may provide the semantics that you need more
+efficiently than Flush Spread.
+<P>
+The
+<I>mbox</I>
+
+should be a pointer to a mailbox variable.  After the 
+<B>FL_connect</B>
+
+call successfully returns, this variable will hold the valid mbox for
+this new connection.
+<P>
+The
+<I>private_name</I>
+
+should be a pointer to a string big enough to hold at least
+MAX_GROUP_NAME characters.  After the call returns it will contain the
+private group name of this connection.  These group names are what are
+reported in membership messages and can be used to send unicast and
+subgroup-multicast messages to this connection.  No other applications
+can join this special group.
+<P>
+<H2>RETURN VALUES</H2>
+
+Returns 
+<B>ACCEPT_SESSION </B>
+
+on success or one of the following errors ( &lt; 0 ):
+<DL COMPACT>
+
+<DT><B>ILLEGAL_SPREAD</B>
+<DD>
+The 
+<I>daemon_name</I>
+given to connect to was illegal for some reason. Usually because
+it was a unix socket on Windows95/NT, an improper format for a host
+or an illegal port number. </p>
+
+<DT><B>COULD_NOT_CONNECT</B>
+
+<DD>
+Lower level socket calls failed to allow a connection to the 
+specified spread daemon right now.</p>
+
+<DT><B>CONNECTION_CLOSED</B>
+
+<DD>
+During communication to establish the connection errors occured
+and the setup could not be completed.</p>
+
+<DT><B>REJECT_VERSION</B>
+
+<DD>
+The daemon and/or libraries have a version mismatch.</p>
+
+<DT><B>REJECT_NO_NAME</B>
+
+<DD>
+No user name was provided.</p>
+
+<DT><B>REJECT_ILLEGAL_NAME</B>
+
+<DD>
+Name provided violated some requirement (length or used an illegal character).</p>
+
+<DT><B>REJECT_NOT_UNIQUE</B>
+
+<DD>
+Name provided is not unique on this daemon. Recommended response is to try
+again with a different name.
+</DL>
+
+<H2>AUTHOR</H2>
+
+John Schultz &lt;<A HREF="mailto:jschultz at cnds.jhu.edu">jschultz at cnds.jhu.edu</A>&gt;
+<P>
+
+<!--#include virtual="/includes/footer" -->
+
+</BODY>
+</HTML>

Added: trunk/docs/flush/docs/FL_disconnect.html
===================================================================
--- trunk/docs/flush/docs/FL_disconnect.html	2005-07-28 14:50:56 UTC (rev 252)
+++ trunk/docs/flush/docs/FL_disconnect.html	2005-07-31 10:51:40 UTC (rev 253)
@@ -0,0 +1,59 @@
+<HTML><HEAD><TITLE>Manpage of FL_disconnect</TITLE>
+</HEAD>
+
+<body bgcolor="#FFFFFF">
+
+<!--#include virtual="/includes/header-a" -->
+
+<a href="http://www.cnds.jhu.edu/research/group/flush_spread">
+<img src="flush_spread_title.gif" alt="FLUSH SPREAD" border=0>
+</a>
+
+<!--#include virtual="/includes/header-b" -->
+
+<H1>FL_disconnect</H1>
+Section: User Manuals (3)<BR>Updated: Dec 2000<BR>
+<HR>
+
+<H2>NAME</H2>
+
+FL_disconnect - destroy a Flush Spread connection between an application and a Spread daemon.
+
+<H2>SYNOPSIS</H2>
+
+<B>#include &lt;fl.h&gt;</B>
+
+<P>
+<B>int FL_disconnect(mailbox </B><I>mbox</I><B>);</B>
+
+<H2>DESCRIPTION</H2>
+
+<B>FL_disconnect</B>
+
+should be called when the application is finished
+with a connection to a Spread daemon.  The application may have
+other connections still open to the daemon and may open a new
+connection after disconnecting. Any data that was available on
+the connection will be lost after the call returns.
+<P>
+The 
+<I>mbox </I>
+
+should be for the connection you wish to close.
+<H2>RETURN VALUES</H2>
+
+Returns 0 on success or 
+<B>ILLEGAL_SESSION</B>
+
+when the
+<I>mbox</I>
+
+given does not represent a valid connection.
+<H2>AUTHOR</H2>
+
+John Schultz &lt;<A HREF="mailto:jschultz at cnds.jhu.edu">jschultz at cnds.jhu.edu</A>&gt;
+
+<!--#include virtual="/includes/footer" -->
+
+</BODY>
+</HTML>

Added: trunk/docs/flush/docs/FL_error.html
===================================================================
--- trunk/docs/flush/docs/FL_error.html	2005-07-28 14:50:56 UTC (rev 252)
+++ trunk/docs/flush/docs/FL_error.html	2005-07-31 10:51:40 UTC (rev 253)
@@ -0,0 +1,42 @@
+<HTML><HEAD><TITLE>Manpage of FL_error</TITLE>
+</HEAD>
+
+<body bgcolor="#FFFFFF">
+
+<!--#include virtual="/includes/header-a" -->
+
+<a href="http://www.cnds.jhu.edu/research/group/flush_spread">
+<img src="flush_spread_title.gif" alt="FLUSH SPREAD" border=0>
+</a>
+
+<!--#include virtual="/includes/header-b" -->
+
+<H1>FL_error</H1>
+Section: User Manuals (3)<BR>Updated: Dec 2000<BR>
+<HR>
+
+<H2>NAME</H2>
+
+FL_error - Flush Spread error string reporting
+<H2>SYNOPSIS</H2>
+
+<B>#include &lt;fl.h&gt;</B>
+
+<P>
+<B>void FL_error(int </B><I>error_code</I><B>);</B>
+
+<H2>DESCRIPTION</H2>
+
+<B>FL_error</B>
+
+prints an error string based on the Flush Spread or Spread error code
+<I>error_code</I>.
+
+<H2>AUTHOR</H2>
+
+John Schultz &lt;<A HREF="mailto:jschultz at cnds.jhu.edu">jschultz at cnds.jhu.edu</A>&gt;
+
+<!--#include virtual="/includes/footer" -->
+
+</BODY>
+</HTML>

Added: trunk/docs/flush/docs/FL_flush.html
===================================================================
--- trunk/docs/flush/docs/FL_flush.html	2005-07-28 14:50:56 UTC (rev 252)
+++ trunk/docs/flush/docs/FL_flush.html	2005-07-31 10:51:40 UTC (rev 253)
@@ -0,0 +1,136 @@
+<HTML><HEAD><TITLE>Manpage of FL_flush</TITLE>
+</HEAD>
+
+<body bgcolor="#FFFFFF">
+
+<!--#include virtual="/includes/header-a" -->
+
+<a href="http://www.cnds.jhu.edu/research/group/flush_spread">
+<img src="flush_spread_title.gif" alt="FLUSH SPREAD" border=0>
+</a>
+
+<!--#include virtual="/includes/header-b" -->
+
+<H1>FL_flush</H1>
+Section: User Manuals (3)<BR>Updated: Dec 2000<BR>
+<HR>
+
+<H2>NAME</H2>
+
+FL_flush - flush a group in response to receiving a flush request message.
+<H2>SYNOPSIS</H2>
+
+<B>#include &lt;fl.h&gt;</B>
+
+<P>
+<B>int FL_flush(mailbox </B><I>mbox</I><B>, const char *</B><I>group_name</I><B>);</B>
+
+<H2>DESCRIPTION</H2>
+
+<B>FL_flush</B>
+
+flushes the group named
+<I>group_name</I>
+
+in which the connection represented by
+<I>mbox</I>
+
+has received a flush request message.  A flush request message is
+delivered in a group when the underlying membership of that group
+changes; it is a signal to the application that the membership has
+changed and some of the original members may have &quot;gone away&quot; (meaning
+that they are no longer in the group with this connection).  However,
+Flush Spread will not install a new membership in the group until each
+of the members of the new membership flush the group.  Flushing a
+group gives Flush Spread the permission of this connection to go ahead
+and install a new view/membership for the group.
+<P>
+During the time after receiving a flush request message but before
+flushing the group the application must be _VERY_ careful about
+calling receive functions; if the application isn't careful enough it
+can permantently block the entire group.  For example, say the
+underlying membership of a group of 10 members changed such that this
+connection is now alone in the group (there is no way an application
+can detect the make-up of a group after receiving a flush request).
+If the application simply calls receive before flushing the group and
+hasn't ensured that it will receive a message somehow (see below) it
+will permanently block itself in the receive and any other members
+that might later be added to that group.
+<P>
+There are a couple ways to avoid this problem, which isn't a bug by
+the way :P, (1) don't call receive functions after receiving a flush
+request message and before flushing the group, (2) call receive but
+set the DONT_BLOCK service flag; if there aren't any messages to
+receive it will break out with a WOULD_BLOCK error, (3) ensure that
+there will be a message to receive somehow.  There are several ways to
+do this, but one 100% sure way to do this is if the application sends
+or has sent a message on this connection (with the SELF_DISCARD
+service _NOT_ set) that it has not yet received back.  In this case,
+the application will eventually receive its own message (even if it
+was to another group) on the connection and can therefore be assured
+of not blocking permanently.
+<P>
+The application _IS_ allowed to send messages to a group after
+receiving a flush request for that group and before flushing the
+group.  However, only a subset of the original members of the current
+view will receive these messages (note that I said a subset, not a
+strict subset).  Technically, this is always the case in Spread: an
+application can determine which other members received certain
+messages for sure by (1) application level message receipt
+acknowledgement, (2) employing the safety properties of SAFE messages
+(although this doesn't guarantee that those members actually processed
+and handled the message), (3) employing the virtual synchrony property
+and the transitional set of the new membership when it is installed
+(although, again, this doesn't guarantee that those members actually
+processed and handled the messages). See FL_receive or read up on
+group communication for more in-depth discussions of this matter.
+<P>
+Anyways, when the application is ready it can flush the group.  Once
+it flushes the group it is not allowed to send any messages to that
+group, until it receives the new membership for that group.  If the
+application breaks this rule, it will receive an ILLEGAL_STATE error.
+<P>
+<H2>RETURN VALUES</H2>
+
+Returns 0 on success or one of the following errors ( &lt; 0 ):
+<DL COMPACT>
+<DT><B>ILLEGAL_SESSION</B>
+
+<DD>
+The connection represented by 
+<I>mbox</I>
+
+is illegal, usually because it is not active.</p>
+
+<DT><B>ILLEGAL_GROUP</B>
+
+<DD>
+The 
+<I>group_name</I>
+
+given to flush was illegal for some reason, usually because it was of
+length 0 or length &gt; MAX_GROUP_NAME.  This error can also be returned
+when a group is flushed for a connection that it has not yet joined or
+is already leaving.</p>
+
+<DT><B>ILLEGAL_STATE</B>
+
+<DD>
+A connection may flush a group only once in response to each flush
+request message delivered in that group.  If you violate this rule you
+will get an ILLEGAL_STATE error.</p>
+
+<DT><B>CONNECTION_CLOSED</B>
+
+<DD>
+Errors occurred during communication and the flush could not be
+initiated.
+</DL>
+<H2>AUTHOR</H2>
+
+John Schultz &lt;<A HREF="mailto:jschultz at cnds.jhu.edu">jschultz at cnds.jhu.edu</A>&gt;
+
+<!--#include virtual="/includes/footer" -->
+
+</BODY>
+</HTML>

Added: trunk/docs/flush/docs/FL_join.html
===================================================================
--- trunk/docs/flush/docs/FL_join.html	2005-07-28 14:50:56 UTC (rev 252)
+++ trunk/docs/flush/docs/FL_join.html	2005-07-31 10:51:40 UTC (rev 253)
@@ -0,0 +1,77 @@
+<HTML><HEAD><TITLE>Manpage of FL_join</TITLE>
+</HEAD>
+
+<body bgcolor="#FFFFFF">
+
+<!--#include virtual="/includes/header-a" -->
+
+<a href="http://www.cnds.jhu.edu/research/group/flush_spread">
+<img src="flush_spread_title.gif" alt="FLUSH SPREAD" border=0>
+</a>
+
+<!--#include virtual="/includes/header-b" -->
+
+<H1>FL_join</H1>
+Section: User Manuals (3)<BR>Updated: Dec 2000<BR>
+<HR>
+
+<H2>NAME</H2>
+
+FL_join - join a connection to a Flush Spread group
+<H2>SYNOPSIS</H2>
+
+<B>#include &lt;fl.h&gt;</B>
+
+<P>
+<B>int FL_join(mailbox </B><I>mbox</I><B>, const char *</B><I>group_name</I><B>);</B>
+
+<H2>DESCRIPTION</H2>
+
+<B>FL_join</B>
+
+joins the connection represented by 
+<I>mbox</I>
+
+to a group with the name
+<I>group_name</I>.
+
+If the group does not exist among the Spread daemons it is created,
+otherwise the connection is joined to the existing group.  A
+connection may not join a group it is already a member of, which it
+has already joined, or from which it is currently leaving.
+<H2>RETURN VALUES</H2>
+
+Returns 0 on success or one of the following errors ( &lt; 0 ):
+<DL COMPACT>
+<DT><B>ILLEGAL_GROUP</B>
+
+<DD>
+The 
+<I>group_name</I>
+
+given to join was illegal for some reason, usually because it was of
+length 0 or length &gt; MAX_GROUP_NAME.  This error is also returned if a
+group with which this connection is already involved (i.e. - already
+joining, already joined, currently leaving) is joined again.</p>
+
+<DT><B>ILLEGAL_SESSION</B>
+
+<DD>
+The connection represented by
+<I>mbox</I>
+
+is illegal, usually because it is not active.</p>
+<DT><B>CONNECTION_CLOSED</B>
+
+<DD>
+Errors occurred during communication and the join could not be
+initiated.</p>
+</DL>
+<H2>AUTHOR</H2>
+
+John Schultz &lt;<A HREF="mailto:jschultz at cnds.jhu.edu">jschultz at cnds.jhu.edu</A>&gt;
+
+<!--#include virtual="/includes/footer" -->
+
+</BODY>
+</HTML>

Added: trunk/docs/flush/docs/FL_leave.html
===================================================================
--- trunk/docs/flush/docs/FL_leave.html	2005-07-28 14:50:56 UTC (rev 252)
+++ trunk/docs/flush/docs/FL_leave.html	2005-07-31 10:51:40 UTC (rev 253)
@@ -0,0 +1,82 @@
+<HTML><HEAD><TITLE>Manpage of FL_leave</TITLE>
+</HEAD>
+
+<body bgcolor="#FFFFFF">
+
+<!--#include virtual="/includes/header-a" -->
+
+<a href="http://www.cnds.jhu.edu/research/group/flush_spread">
+<img src="flush_spread_title.gif" alt="FLUSH SPREAD" border=0>
+</a>
+
+<!--#include virtual="/includes/header-b" -->
+
+<H1>FL_leave</H1>
+Section: User Manuals (3)<BR>Updated: Dec 2000<BR>
+<HR>
+
+<H2>NAME</H2>
+
+FL_leave - remove a connection from a Flush Spread group
+<A NAME="lbAC">&nbsp;</A>
+<H2>SYNOPSIS</H2>
+
+<B>#include &lt;fl.h&gt;</B>
+
+<P>
+<B>int FL_leave(mailbox </B><I>mbox</I><B>, const char *</B><I>group_name</I><B>);</B>
+
+<H2>DESCRIPTION</H2>
+
+<B>FL_leave</B>
+
+removes the connection represented by
+<I>mbox</I>
+
+from a Flush Spread group with the name
+<I>group_name</I>.
+
+A connection may only leave a group after it has been installed as a
+member of the group by being included in a Flush Spread
+view/membership.  Also a connection may not leave a group if it has
+already requested to leave that group and the self-leave message has
+not yet been received.
+<H2>RETURN VALUES</H2>
+
+Returns 0 on success or one of the following errors ( &lt; 0 ):
+<DL COMPACT>
+<DT><B>ILLEGAL_GROUP</B>
+
+<DD>
+The 
+<I>group_name</I>
+
+given to leave was illegal for some reason. Usually because it was of
+length 0 or length &gt; MAX_GROUP_NAME.  This error is also returned if
+the connection was not a full-fledged member (i.e. - not yet included
+in a Flush membership message for that group, or already leaving) of
+the group.</p>
+
+<DT><B>ILLEGAL_SESSION</B>
+
+<DD>
+The connection represented by 
+<I>mbox</I>
+
+is illegal. Usually because it is not active.</p>
+
+<DT><B>CONNECTION_CLOSED</B>
+
+<DD>
+Errors occurred during communication
+and the leave could not be initiated.
+</DL>
+
+<H2>AUTHOR</H2>
+
+John Schultz &lt;<A HREF="mailto:jschultz at cnds.jhu.edu">jschultz at cnds.jhu.edu</A>&gt;
+
+<!--#include virtual="/includes/footer" -->
+
+</BODY>
+</HTML>

Added: trunk/docs/flush/docs/FL_more_msgs.html
===================================================================
--- trunk/docs/flush/docs/FL_more_msgs.html	2005-07-28 14:50:56 UTC (rev 252)
+++ trunk/docs/flush/docs/FL_more_msgs.html	2005-07-31 10:51:40 UTC (rev 253)
@@ -0,0 +1,79 @@
+<HTML><HEAD><TITLE>Manpage of FL_more_msgs</TITLE>
+</HEAD>
+
+<body bgcolor="#FFFFFF">
+
+<!--#include virtual="/includes/header-a" -->
+
+<a href="http://www.cnds.jhu.edu/research/group/flush_spread">
+<img src="flush_spread_title.gif" alt="FLUSH SPREAD" border=0>
+</a>
+
+<!--#include virtual="/includes/header-b" -->
+
+<H1>FL_more_msgs</H1>
+Section: User Manuals (3)<BR>Updated: Dec 2000<BR>
+<HR>
+
+<H2>NAME</H2>
+
+FL_more_msgs - returns the number of complete messages buffered and
+ready to be received on a connection.
+<H2>SYNOPSIS</H2>
+
+<B>#include &lt;fl.h&gt;</B>
+
+<P>
+<B>int FL_more_msgs(mailbox </B><I>mbox</I><B>);</B>
+
+<H2>DESCRIPTION</H2>
+
+<B>FL_more_msgs</B>
+
+allows an application to check to see if any complete messages are
+already buffered and ready to be received on the connection represented by
+<I>mbox</I>.
+
+<P>
+<B>NOTE</B>,
+
+that this function 
+<B>CANNOT </B>
+
+be used as an I/O polling function to check if a receive call should
+be done.  If this function returns zero there still might be a message
+ready to receive on the connection: FL_poll, and file descriptor
+selects/polls can detect if there is activity (not necessarily a
+message) on a connection, while DONT_BLOCK receive semantics can
+detect whether or not there is a message on the connection and
+FL_more_msgs can only answer if there are any buffered messages
+already on the connection.
+<P>
+This function is merely a helper function to re-get the current status
+of the
+<I>more_messes</I>
+
+parameter returned from a call to 
+<B>FL_receive</B>.
+
+<H2>RETURN VALUES</H2>
+
+Returns the number of complete buffered messages ready to be received,
+or one of the following errors ( &lt; 0):
+<DL COMPACT>
+<DT><B>ILLEGAL_SESSION</B>
+
+<DD>
+The connection represented by 
+<I>mbox</I>
+
+is illegal, usually because it is not active.
+</DL>
+<H2>AUTHOR</H2>
+
+John Schultz &lt;<A HREF="mailto:jschultz at cnds.jhu.edu">jschultz at cnds.jhu.edu</A>&gt;
+
+<!--#include virtual="/includes/footer" -->
+
+</BODY>
+</HTML>

Added: trunk/docs/flush/docs/FL_multicast.html
===================================================================
--- trunk/docs/flush/docs/FL_multicast.html	2005-07-28 14:50:56 UTC (rev 252)
+++ trunk/docs/flush/docs/FL_multicast.html	2005-07-31 10:51:40 UTC (rev 253)
@@ -0,0 +1,242 @@
+<HTML><HEAD><TITLE>Manpage of FL_multicast</TITLE>
+</HEAD>
+
+<body bgcolor="#FFFFFF">
+
+<!--#include virtual="/includes/header-a" -->
+
+<a href="http://www.cnds.jhu.edu/research/group/flush_spread">
+<img src="flush_spread_title.gif" alt="FLUSH SPREAD" border=0>
+</a>
+
+<!--#include virtual="/includes/header-b" -->
+
+<H1>FL_multicast</H1>
+Section: User Manuals (3)<BR>Updated: Dec 2000<BR>
+<HR>
+
+<H2>NAME</H2>
+
+FL_unicast, FL_scat_unicast, FL_subgroupcast, FL_scat_subgroupcast, FL_multicast, FL_scat_multicast - multicast messages to subsets of Flush Spread groups.
+<H2>SYNOPSIS</H2>
+
+<B>#include &lt;fl.h&gt;</B>
+
+<P>
+<B>int FL_unicast(mailbox </B><I>mbox</I><B>, service </B><I>serv_type</I><B>, const char *</B><I>group_name</I><B>, const char *</B><I>recvr_name</I><B>, int16 </B><I>mess_type</I><B>, int </B><I>mess_len</I><B>, const char *</B><I>mess</I><B>);</B>
+
+<P>
+<B>int FL_scat_unicast(mailbox </B><I>mbox</I><B>, service </B><I>serv_type</I><B>, const char *</B><I>group_name</I><B>, const char *</B><I>recvr_name</I><B>, int16 </B><I>mess_type</I><B>, const scatter *</B><I>scat</I><B>);</B>
+
+<P>
+<B>int FL_subgroupcast(mailbox </B><I>mbox</I><B>, service </B><I>serv_type</I><B>, const char *</B><I>group_name</I><B>, int </B><I>num_recvrs</I><B>, const char </B><I>recvr_names</I><B>[][MAX_GROUP_NAME], int16 </B><I>mess_type</I><B>, int </B><I>mess_len</I><B>, const char *</B><I>mess</I><B>);</B>
+
+<P>
+<B>int FL_scat_subgroupcast(mailbox </B><I>mbox</I><B>, service </B><I>serv_type</I><B>, const char *</B><I>group_name</I><B>, int </B><I>num_recvrs</I><B>, const char </B><I>recvr_names</I><B>[][MAX_GROUP_NAME], int16 </B><I>mess_type</I><B>, const char *</B><I>scat</I><B>);</B>
+
+<P>
+<B>int FL_unicast(mailbox </B><I>mbox</I><B>, service </B><I>serv_type</I><B>, const char *</B><I>group_name</I><B>, int16 </B><I>mess_type</I><B>, int </B><I>mess_len</I><B>, const char *</B><I>mess</I><B>);</B>
+
+<P>
+<B>int FL_scat_unicast(mailbox </B><I>mbox</I><B>, service </B><I>serv_type</I><B>, const char *</B><I>group_name</I><B>, int16 </B><I>mess_type</I><B>, const scatter *</B><I>scat</I><B>);</B>
+
+<H2>DESCRIPTION</H2>
+
+<B>FL_multicast</B>
+
+and its variants all multicast a message from the connection represented by
+<I>mbox</I>
+
+to a subset of the membership of the Flush Spread group named
+<I>group_name</I>.
+
+<P>
+Note, that unlike Spread, Flush Spread does not provide guarantees
+about messages sent to different groups; meaning that FIFO, CAUSAL,
+SAFE, etc. safety properties are not maintained across different Flush
+Spread groups, but only within a group to which they are sent. For
+example, if an application sends a FIFO message A to the private group
+G and then sends another FIFO message B to the regular group H, of
+which G is a member, then G could receive B before A or vice versa.
+<P>
+In order to multicast a message to a group, the destination group 
+<I>group_name</I>
+
+must either be a private group name, or this connection must be a
+full-fledged member of that regular group (has joined and been
+included in a membership of the group, and is not leaving the group,
+see FL_join, FL_leave) and not be in an illegal state to multicast (by
+having recently flushed the group and not yet received its next
+view/membership, see FL_flush).
+<P>
+The
+<I>serv_type</I>
+
+is a type field that should be set to the service type this message
+requires. The valid flags for multicasting messages are:
+<P>
+<DL COMPACT><DT><DD>
+<B>UNRELIABLE_MESS</B>
+
+<BR>
+
+
+<B>RELIABLE_MESS </B>
+
+<BR>
+
+
+<B>FIFO_MESS</B>
+
+<BR>
+
+
+<B>CAUSAL_MESS</B>
+
+<BR>
+
+
+<B>AGREED_MESS</B>
+
+<BR>
+
+
+<B>SAFE_MESS</B>
+
+</DL>
+
+<P>
+This type field can be bit-wise ORed with other flags like
+SELF_DISCARD if desired.  Currently SELF_DISCARD is the only
+additional flag for multicasts: SELF_DISCARD will cause the
+multicasted message to NOT be delivered back to this connection.
+<P>
+The string
+<I>group_name</I>
+
+indicates the name of the Flush Spread group to which this message
+should be sent.  This group can be either a regular or a private Flush
+Spread group, as dicussed above.  The multicast variants sends the
+message to all members of a group, while the unicast and subgroupcast
+variants allow the sender to specify a subset of the group to which to
+send the message.  For the unicast variants the sender specifies the
+private group name of the intended recipient in the variable
+<I>recvr_name</I>.
+
+For the subgroupcast variants the sender specifies the number of
+recipients in the variable
+<I>num_recvrs</I>
+
+and a doubly scripted array of the members' private group names in the
+variable
+<I>recvr_names</I>,
+
+where duplicate names are tolerated.  For the unicast and subgroupcast
+variants all specified receivers must have been members of the most
+recently installed view/membership of the group.  If the application
+breaks this rule it will receive an ILLEGAL_RECEIVERS error.
+<P>
+The
+<I>mess_type</I>
+
+is a 16 bit integer which can be used by the application arbitrarily.
+However, Flush Spread restricts which values can be used to be greater
+than or equal to FL_MIN_LEGAL_MESS_TYPE.  If the application sends
+with a message type of less than FL_MIN_LEGAL_MESS_TYPE, then the
+error ILLEGAL_MESSAGE_TYPE will be returned.  The intent of this
+message type is that the application can use it to name different
+kinds of data messages so they can be differentiated without looking
+into the body of the message.  This value
+<B>WILL</B>
+
+be endian corrected upon returning. 
+<P>
+The non-scatter variants use a single buffer to pass the body of the
+message to be sent.  The
+<I>mess_len</I>
+
+field gives the message length in bytes. While the
+<I>mess</I>
+
+field is a pointer to the buffer containing the message.  For the
+scatter variants, both of these parameters are replaced with one
+pointer,
+<I>scat_mess</I>,
+
+to a scatter structure, which is similiar to an iovec.  This allows
+messages made up of several parts to be sent without an extra copy on
+systems which support scatter-gather.  However, the number of scatter
+elements is restricted to be non-negative and less than or equal to
+FL_MAX_SCATTER_ELEMENTS.  If a buffer length is negative or an illegal
+number of scatter elements is used then ILLEGAL_MESSAGE will be
+returned.
+
+<H2>RETURN VALUES</H2>
+
+Returns the number of bytes sent on success or one of the following
+errors ( &lt; 0 ):
+<DL COMPACT>
+<DT><B>ILLEGAL_SESSION</B>
+
+<DD>
+The connection represented by 
+<I>mbox</I>
+
+was illegal, usually because it is no longer active.</p>
+
+<DT><B>ILLEGAL_GROUP</B>
+
+<DD>
+The 
+<I>group_name</I>
+
+given to multicast to was illegal for some reason. Usually because it
+was of length 0 or length &gt; MAX_GROUP_NAME.  This error can also be
+returned if the connection was not a full-fledged member (i.e. - not
+yet included in a Flush membership message for the group, or already
+leaving) of a regular group.</p>
+
+<DT><B>ILLEGAL_RECEIVERS</B>
+
+<DD>
+A unicast or subgroupcast specified private group names that weren't
+members of
+<I>group_name</I>'s
+
+most recent view/membership.</p>
+
+<DT><B>ILLEGAL_STATE</B>
+
+<DD>
+A multicast to a group was attempted after flushing the group and
+before receiving the next view/membership for that group.</p>
+
+<DT><B>ILLEGAL_SERVICE</B>
+
+<DD>
+If the service type was using bits reserved by Flush Spread or Spread.</p>
+
+<DT><B>ILLEGAL_PARAM</B>
+
+<DD>
+An illegal parameter, such as a negative array size was passed.</p>
+
+<DT><B>ILLEGAL_MESSAGE</B>
+
+<DD>
+The message had an illegal structure, like a negative buffer size or
+an illegal number of scatter elements.</p>
+
+<DT><B>CONNECTION_CLOSED</B>
+
+<DD>
+Errors occurred during communication and the multicast could not be completed.
+</DL>
+<H2>AUTHOR</H2>
+
+John Schultz &lt;<A HREF="mailto:jschultz at cnds.jhu.edu">jschultz at cnds.jhu.edu</A>&gt;
+
+<!--#include virtual="/includes/footer" -->
+
+</BODY>
+</HTML>

Added: trunk/docs/flush/docs/FL_poll.html
===================================================================
--- trunk/docs/flush/docs/FL_poll.html	2005-07-28 14:50:56 UTC (rev 252)
+++ trunk/docs/flush/docs/FL_poll.html	2005-07-31 10:51:40 UTC (rev 253)
@@ -0,0 +1,68 @@
+<HTML><HEAD><TITLE>Manpage of FL_poll</TITLE>
+</HEAD>
+
+<body bgcolor="#FFFFFF">
+
+<!--#include virtual="/includes/header-a" -->
+
+<a href="http://www.cnds.jhu.edu/research/group/flush_spread">
+<img src="flush_spread_title.gif" alt="FLUSH SPREAD" border=0>
+</a>
+
+<!--#include virtual="/includes/header-b" -->
+
+<H1>FL_poll</H1>
+Section: User Manuals (3)<BR>Updated: Dec 2000<BR>
+<HR>
+
+<H2>NAME</H2>
+
+FL_poll - returns the amount, in bytes, of activity on a connection.
+<H2>SYNOPSIS</H2>
+
+<B>#include &lt;fl.h&gt;</B>
+
+<P>
+<B>int FL_poll(mailbox </B><I>mbox</I><B>);</B>
+
+<H2>DESCRIPTION</H2>
+
+<B>FL_poll</B>
+
+is a way to poll the connection represented by
+<I>mbox</I>
+
+to see if there is any activity on that connection.  
+<P>
+<B>NOTE</B>,
+
+however that activity does
+<B>NOT</B>
+
+necessarily mean that a message is available to be read, as in Spread.
+Only a call to FL_more_msgs that returns a positive number of buffered
+messages or a DONT_BLOCK receive call can check for sure to see if a
+message is now available to be read.  If you don't use either of these
+semantics, then you may enter into a blocking receive call, which can
+sometimes be dangerous (see FL_flush).
+<H2>RETURN VALUES</H2>
+
+Returns the number of bytes of activity currently on the connection,
+or one of the following errors ( &lt; 0 ):
+<DL COMPACT>
+<DT><B>ILLEGAL_SESSION</B>
+
+<DD>
+The connection represented by 
+<I>mbox</I>
+
+is illegal, usually because it is not active.
+</DL>
+<H2>AUTHOR</H2>
+
+John Schultz &lt;<A HREF="mailto:jschultz at cnds.jhu.edu">jschultz at cnds.jhu.edu</A>&gt;
+
+<!--#include virtual="/includes/footer" -->
+
+</BODY>
+</HTML>

Added: trunk/docs/flush/docs/FL_receive.html
===================================================================
--- trunk/docs/flush/docs/FL_receive.html	2005-07-28 14:50:56 UTC (rev 252)
+++ trunk/docs/flush/docs/FL_receive.html	2005-07-31 10:51:40 UTC (rev 253)
@@ -0,0 +1,522 @@
+<HTML><HEAD><TITLE>Manpage of FL_receive</TITLE>
+</HEAD>
+
+<body bgcolor="#FFFFFF">
+
+<!--#include virtual="/includes/header-a" -->
+
+<a href="http://www.cnds.jhu.edu/research/group/flush_spread">
+<img src="flush_spread_title.gif" alt="FLUSH SPREAD" border=0>
+</a>
+
+<!--#include virtual="/includes/header-b" -->
+
+<BODY>
+<H1>FL_receive</H1>
+Section: User Manuals (3)<BR>Updated: Dec 2000<BR>
+<HR>
+
+<H2>NAME</H2>
+
+Fl_receive, FL_scat_receive - receive a message on a Flush Spread connection.
+<H2>SYNOPSIS</H2>
+
+<B>#include &lt;fl.h&gt;</B>
+
+<P>
+<B>int FL_receive(mailbox </B><I>mbox</I><B>, service *</B><I>serv_type</I><B>, char </B><I>sender</I><B>[MAX_GROUP_NAME], int </B><I>max_groups</I><B>, int *</B><I>num_groups</I><B>, char </B><I>groups</I><B>[][MAX_GROUP_NAME], int16 *</B><I>mess_type</I><B>, int *</B><I>endian_mismatch</I><B>, int </B><I>max_mess_len</I><B>, char *</B><I>mess</I><B>, int *</B><I>more_messes</I><B>);</B>
+
+<P>
+<B>int FL_scat_receive(mailbox </B><I>mbox</I><B>, service *</B><I>serv_type</I><B>, char </B><I>sender</I><B>[MAX_GROUP_NAME], int </B><I>max_groups</I><B>, int *</B><I>num_groups</I><B>, char </B><I>groups</I><B>[][MAX_GROUP_NAME], int16 *</B><I>mess_type</I><B>, int *</B><I>endian_mismatch</I><B>, scatter *</B><I>scat_mess</I><B>, int *</B><I>more_messes</I><B>);</B>
+
+<H2>DESCRIPTION</H2>
+
+<B>FL_receive</B>
+
+is the general purpose message receipt function for Flush Spread.
+Messages for all groups joined on this connection will arrive on the
+same mailbox.  A call to
+<B>FL_receive</B>
+
+will perform a receive on a connection to get a single message from
+any one of the groups to which it is joined.  
+<P>
+After a call to receive completes, a number of the passed fields are
+set to values indicating meta-information about the message (such as
+destination group, message type, endianness, etc).  The meanings of
+these different meta fields depends on the type of message received
+and the receipt services requested.
+<P>
+Input Parameters:
+<DL COMPACT><DT><DD>
+<DL COMPACT>
+<DT><I>mbox</I>
+
+<DD>
+Represents the connection on which to receive a message.</p>
+<DT><I>serv_type</I>
+
+<DD>
+A pointer to a service bit field, requesting some receive services.
+This field can have the DONT_BLOCK and/or DROP_RECV service bit flags
+flipped on.  This bit field should be zeroed out before each call if
+no special receive service is to be requested.</p>
+<DT><I>sender</I>
+
+<DD>
+A pointer to a character array with storage for at least
+MAX_GROUP_NAME characters.</p>
+<DT><I>max_groups</I>
+
+<DD>
+A non-negative int representing how many group names the application
+is willing to receive in this receive call.</p>
+<DT><I>num_groups</I>
+
+<DD>
+A pointer to an int.</p>
+<DT><I>groups</I>
+
+<DD>
+An array of group names containing storage for at least 
+<I>max_groups</I>
+
+* MAX_GROUP_NAME characters.</p>
+
+<DT><I>mess_type</I>
+
+<DD>
+A pointer to an int16.</p>
+
+<DT><I>endian_mismatch</I>
+
+<DD>
+A pointer to an int.</p>
+
+<DT><I>max_mess_len</I>
+
+<DD>
+A non-negative integer representing how many bytes of data the
+application is willing to receive in this call.</p>
+
+<DT><I>mess</I>
+
+<DD>
+A pointer to a buffer containing storage for at least
+<I>max_mess_len</I>
+
+bytes.</p>
+
+<DT><I>scat_mess</I>
+
+<DD>
+A pointer to a scatter (a cousin of an iovec) with a non-negative
+number of scatter elements that is less than or equal to
+MAX_SCATTER_ELEMENTS (that's not a typo :), and all non-negative
+buffer lengths in the indicated scatter elements to get scatter-gather
+semantics.
+</DL></DL>
+
+<P>
+Receive Semantics:
+<P>
+Normally, a call to receive will block if no messages are immediately
+available.
+<P>
+Normally, when calling a receive function if a user's buffer
+(<I>groups</I>,
+
+<I>mess</I>,
+
+or 
+<I>scat_mess</I>)
+
+is too small to contain the data to be returned, then a
+GROUPS_TOO_SHORT or BUFFER_TOO_SHORT error code will be returned.  In
+this case, the
+<I>serv_type</I>,
+
+and 
+<I>mess_type</I>
+
+of the message are filled in from the offending message and the
+parameters
+<I>num_groups</I>,
+
+and 
+<I>endian_mismatch</I>
+
+reflect information about the necessary buffers' sizes.  If
+<I>max_groups</I>
+
+was big enough, then
+*<I>num_groups</I>
+
+will be zero, otherwise it will be the negative of how many group
+names are available to be received (i.e. -7 means the 
+<I>max_groups</I>
+
+was less than 7 and 7 group names can be received).  If
+<I>max_mess_len</I>
+
+or
+<I>scat_mess</I>
+
+was big enough then
+*<I>endian_mismatch</I>
+
+will be zero, otherwise it will be the negative of how much data is
+available to be received (i.e. -32768 means the msg buffer was too
+small and 32768 bytes of data can be received).  The offending message
+is still available to be received through later calls to receive with
+appropriately sized user buffers.
+<P>
+The parameter
+<I>serv_type</I>
+
+allows the applications to request different receive services that
+affect the normal semantics of receive.  Currently, the only services
+are DONT_BLOCK and DROP_RECV.
+<P>
+The DONT_BLOCK service makes a receive call non-blocking.  With this
+service, the receive call will return quickly either with a message or
+with the error code WOULD_BLOCK if no message is available.  Using
+this service is the only way to detect if a message is ready on the
+connection in a non-blocking manner in Flush Spread.
+<P>
+The DROP_RECV service forces Flush Spread to read the current message
+off of the connection regardless of whether or not the user's buffers
+are big enough to fit all of the data.  In this case, a TOO_SHORT
+error code is still returned even though the call actually
+succeeded. Anyway, as much data as can be fit into the user's buffers
+will be stuffed into them, and that message will no longer be on the
+connection.  Also,
+*<I>num_groups</I>
+
+will still be set to the negative size of how many names were
+available (if 
+<I>groups</I>
+
+wasn't big enough), but
+*<I>endian_mismatch</I>
+
+will not reflect the size of the data that could have been received (if
+<I>mess</I>
+
+or
+<I>scat_mess</I>
+
+wasn't big enough), instead it indicates whether or not the sending
+machine had an opposite endianness or not.  Using DROP_RECV in this
+manner, when a buffer problem occurs there is no way to determine how
+big the data portion of the message actually was.  This service is
+meant to be used when a call to receive without DROP_RECV fails with a
+buffer error, but it is determined that the message isn't all that
+important, or something along those lines; otherwise reallocate your
+receive buffers appropriately and re-call receive again without
+the DROP_RECV service.
+<P>
+Output Parameters:
+<P>
+Upon a successful return from receive, 
+*<I>serv_type</I>
+
+will be filled out with the message type that was just received.  The
+specific type of message received can be tested using the different
+message and membership type access macros.  The rest of the
+parameters' meanings differ depending on the
+<I>serv_type</I>.
+
+<P>
+Regular Messages:
+<P>
+If the
+<I>serv_type</I>
+
+is a REG_MESSAGE (i.e. a data message) then:
+<P>
+The parameter
+<I>sender</I>
+
+will be filled with the private group name of the sending connection.
+<P>
+The parameter
+<I>num_groups</I>
+
+will be set to the number of group names filled into
+<I>groups</I>.
+
+<P>
+The 
+<I>groups</I>
+
+array will contain the group to which this message was sent.  If the
+message is a SUBGROUP_CAST (check the service type, see FL_multicast)
+then all of the recipients' private group names will be listed and the
+last name in the array (i.e.
+groups[*<I>num_groups</I>-1])
+
+will be the regular group to which this message was sent.
+<P>
+If DROP_RECV is being used, and 
+<I>groups</I>
+
+is too small, then as many names as can fit will be filled in as above
+described.  If the message was a SUBGROUP_CAST, then the last group in
+the array (i.e.
+groups[<I>max_groups</I>-1])
+
+will be the destination group for the message. If 
+<I>max_groups</I>
+
+is zero then even the destination group is not reported.
+<P>
+The parameter
+*<I>mess_type</I>
+
+will be set to the message type field the application sent with
+the original message, which is a restricted (see FL_multicast) 16 bit
+integer.  This value is endian corrected upon returning.
+<P>
+The parameter
+*<I>endian_mismatch</I>
+
+will be set to true (non-zero) if the endianness of the sending
+machine is the opposite of this receiving machine's.
+<P>
+The actual message body being received will be filled into either the
+<I>mess</I>
+
+or
+<I>scat_mess</I>
+
+parameter.
+<P>
+Flush Request Message:
+<P>
+If this is a FLUSH_REQ_MESS then this represents that the underlying
+membership has changed and that this application needs to flush this
+group before the new view/membership can be installed (see
+FL_flush). 
+<P>
+The
+<I>sender</I>
+
+will be filled in with the name of the group this connection needs
+to flush.  All the other fields are set to empty values.
+<P>
+Membership Messages:
+<P>
+If this is a MEMB_MESSAGE (i.e. membership message) and it
+specifically is a REG_MEMB_MESS, then:
+<P>
+The 
+<I>sender</I>
+
+will be filled with the name of the group for which the membership
+change is occuring.
+<P>
+The
+<I>mess_type</I>
+
+field will be set to the index of this process in the 
+<I>groups</I>
+
+array (see below). 
+<P>
+The
+<I>endian_mismatch</I>
+
+field will be set to 0 since there are no endian issues with regular memberships.
+<P>
+The
+<I>groups</I>
+
+array and message
+body are used to provide two kinds of membership information about the change that just
+occured in this group.  The 
+<I>num_groups</I>
+
+field will be set to the number of members in the group in the 
+<B>new</B>
+
+membership (i.e. after the change occured). Correspondingly, the 
+<I>groups</I>
+
+array will be filled in with the private group names of all members of this
+group in the
+<B>new</B>
+
+membership.  This list of names is always in the same order at all receipients
+and thus can be used to deterministically pick a group representative if
+one is needed by the application.
+<P>
+The second set of information is stored in the message body.  The data
+buffer will include the following fields:
+<DL COMPACT><DT><DD>
+<DL COMPACT>
+<DT><B>group_id vid;</B>
+
+<DD>
+<BR>
+
+<DT><B>int num_members;</B>
+
+<DD>
+<BR>
+
+<DT><B>char dgroups[][MAX_GROUP_NAME];</B>
+
+<DD>
+</DL></DL>
+
+<P>
+The vid and num_members parameters will already be endian corrected.
+The dgroups array will contain num_members group names.  The content
+of the dgroups array is dependent upon the type of the membership
+change:
+<DL COMPACT><DT><DD>
+<DL COMPACT>
+<DT><B>CAUSED_BY_JOIN:</B>
+
+<DD>
+dgroups contains the private group of the joining process.</p>
+<DT><B>CAUSED_BY_LEAVE:</B>
+
+<DD>
+dgroups contains the private group of the leaving process.</p>
+<DT><B>CAUSED_BY_DISCONNECT:</B>
+
+<DD>
+dgroups contains the private group of the disconnecting process.</p>
+<DT><B>CAUSED_BY_NETWORK:</B>
+
+<DD>
+dgroups contains the group names of the members of the new membership who came 
+with 
+<B>this</B>
+
+application to the new view/membership.  Using this data the previous
+membership listing and the new membership listing an application can
+determine all the members that left and were added in the NETWORK
+membership change.  Note, that a member can both leave and be added in
+a 
+<B>single</B>
+
+NETWORK change.
+</DL></DL>
+
+<P>
+Transitional Message:
+<P>
+If this is a MEMB_MESSAGE and specifically it is a TRANSITION_MESS,
+then this means that one or more of the current members of the group
+has left and not all of the safety guarantees can be met w.r.t. all of
+the original members for the messages delivered after this signal and
+before the next view/membership.  Transitional messages never have a
+CAUSED_BY type because they only serve to signal up to where SAFE
+delivery and AGREED delivery (with no holes) is guaranteed in the
+view/membership in which they are delivered.  Only one TRANSITION_MESS
+is delivered per group view/membership per connection.
+<P>
+The 
+<I>sender</I>
+
+will be filled in with the name of the group for which the membership
+change is occurring.  All the other parameters will be set to empty values.
+<P>
+Self-Leave Message:
+<P>
+If this is a MEMB_MESSAGE (i.e. membership message) and it is neither
+a REG_MEMB_MESS or a TRANSITION_MESS, then it represents exactly the
+situtation where the member receiving this message has left a group
+and this is notification that the leave has occured, thus it is
+sometimes called a self-leave message.  The simplest test for this is
+if a message is CAUSED_BY_LEAVE and REG_MEMB_MESS is false (zero),
+then it is a self-leave message.
+<P>
+The other members of the group that this member just left will receive
+a normal TRANSITION_MESS, REG_MEMB_MESS pair as described above
+showing this connection leaving.
+<P>
+For self-leave messages the
+<I>sender</I>
+
+field will be filled in with the name of the group this connection is
+leaving.  All the other fields are set to empty values.
+<P>
+<H2>RETURN VALUES</H2>
+
+Returns the size of the data portion of the message received on
+success or one of the following errors ( &lt; 0 ):
+<DL COMPACT>
+<DT><B>ILLEGAL_SESSION</B>
+
+<DD>
+The connection represented by 
+<I>mbox</I>
+
+was illegal, usually because it is not active.</p>
+
+<DT><B>ILLEGAL_PARAM</B>
+
+<DD>
+An illegal parameter was passed, like a negative array size.</p>
+
+<DT><B>ILLEGAL_MESSAGE</B>
+
+<DD>
+The msg buffer had an illegal structure, like an illegal number of
+scatter elements or a negatively sized buffer.</p>
+
+<DT><B>WOULD_BLOCK</B>
+
+<DD>
+The receive call was made with the DONT_BLOCK service and it would
+have blocked because no messages were ready to be received.</p>
+
+<DT><B>GROUPS_TOO_SHORT</B>
+
+<DD>
+The receive call was made with a groups array that was too small.  
+*<I>num_groups</I>
+
+is filled in with the negative number of names that could have been
+returned.  Note, that the msg buffer could also be too short, and this
+should be determined by examining
+<I>endian_mismatch</I>
+
+as described above. If DROP_RECV was used this error code actually
+represents success (see above).</p>
+
+<DT><B>BUFFER_TOO_SHORT</B>
+
+<DD>
+The receive call was made with a msg buffer that was too small.
+*<I>endian_mismatch</I>
+
+is filled in with the negative number of bytes that could have been
+received if DROP_RECV isn't used.  If it is used then this error code
+actually represents success and 
+<I>endian_mismatch</I>
+
+has the normal meaning (see above).  Note, that the groups buffer
+could also be too short, this should be determined by examining
+<I>num_groups</I>
+
+as described above.</p>
+
+<DT><B>CONNECTION_CLOSED</B>
+
+<DD>
+During communication to receive, communication errors occurred
+and the receive could not be completed.
+</DL>
+<H2>AUTHOR</H2>
+
+John Schultz &lt;<A HREF="mailto:jschultz at cnds.jhu.edu">jschultz at cnds.jhu.edu</A>&gt;
+
+<!--#include virtual="/includes/footer" -->
+
+</BODY>
+</HTML>

Added: trunk/docs/flush/docs/FL_version.html
===================================================================
--- trunk/docs/flush/docs/FL_version.html	2005-07-28 14:50:56 UTC (rev 252)
+++ trunk/docs/flush/docs/FL_version.html	2005-07-31 10:51:40 UTC (rev 253)
@@ -0,0 +1,44 @@
+<HTML><HEAD><TITLE>Manpage of FL_version</TITLE>
+</HEAD>
+
+<body bgcolor="#FFFFFF">
+
+<!--#include virtual="/includes/header-a" -->
+
+<a href="http://www.cnds.jhu.edu/research/group/flush_spread">
+<img src="flush_spread_title.gif" alt="FLUSH SPREAD" border=0>
+</a>
+
+<!--#include virtual="/includes/header-b" -->
+
+<H1>FL_version</H1>
+Section: User Manuals (3)<BR>Updated: Dec 2000<BR>
+<HR>
+
+<H2>NAME</H2>
+
+FL_version - returns library version information.
+<A NAME="lbAC">&nbsp;</A>
+<H2>SYNOPSIS</H2>
+
+<B>#include &lt;fl.h&gt;</B>
+
+<P>
+<B>void FL_version(int *</B><I>major_ver</I><B>, int *</B><I>minor_ver</I><B>, int *</B><I>patch_ver</I><B>);</B>
+
+<A NAME="lbAD">&nbsp;</A>
+<H2>DESCRIPTION</H2>
+
+<B>FL_version</B>
+
+returns the major, minor, and patch version numbers of the Flush
+Spread library in use.
+<A NAME="lbAE">&nbsp;</A>
+<H2>AUTHOR</H2>
+
+John Schultz &lt;<A HREF="mailto:jschultz at cnds.jhu.edu">jschultz at cnds.jhu.edu</A>&gt;
+
+<!--#include virtual="/includes/footer" -->
+
+</BODY>
+</HTML>

Added: trunk/docs/flush/docs/access.html
===================================================================
--- trunk/docs/flush/docs/access.html	2005-07-28 14:50:56 UTC (rev 252)
+++ trunk/docs/flush/docs/access.html	2005-07-31 10:51:40 UTC (rev 253)
@@ -0,0 +1,42 @@
+<html>
+<head><title>Flush Documentation: Message Type Access Macros</title></head>
+
+<body bgcolor="#FFFFFF">
+
+<!--#include virtual="/includes/header-a" -->
+
+<a href="http://www.cnds.jhu.edu/research/group/flush_spread">
+<img src="flush_spread_title.gif" alt="FLUSH SPREAD" border=0>
+</a>
+
+<!--#include virtual="/includes/header-b" -->
+
+<h2><em>Message Type Access Macros</em></h2>
+
+Flush Spread uses the same message type access macros as Spread. In addition to these
+<a href="http://www.spread.org/docs/access.html">common macros</a>, Flush Spread also
+adds the following message type access macros:
+
+<pre>
+#define Is_flush_req_mess(serv_type)
+#define Is_subgroup_mess(serv_type)
+</pre>
+
+<ol>
+<li><tt>Is_flush_req_mess(serv_type):</tt></p>
+
+This macro checks the message type to determine if the message is a flush request message or not.
+Flush request messages are an important part of the view synchrony GCS semantics, see <a
+href="FL_flush.html">FL_flush</a> for an in-depth discussion of them.</p>
+
+<li><tt>Is_subgroup_mess(serv_type):</tt></p>
+
+This macro checks the message type to determine if the message is a subgroup-multicast or
+not. A subgroup-multicast is a message that is only sent to a subset of a group, see
+<a href="FL_multicast.html">FL_subgroupcast</a> for a better description.</p>
+
+</ol>
+
+<!--#include virtual="/includes/footer" -->
+
+</body></html>

Added: trunk/docs/flush/docs/error.html
===================================================================
--- trunk/docs/flush/docs/error.html	2005-07-28 14:50:56 UTC (rev 252)
+++ trunk/docs/flush/docs/error.html	2005-07-31 10:51:40 UTC (rev 253)
@@ -0,0 +1,80 @@
+<html>
+<head><title>Flush Documentation: Error Return Codes</title></head>
+
+<body bgcolor="#FFFFFF">
+
+<!--#include virtual="/includes/header-a" -->
+
+<a href="http://www.cnds.jhu.edu/research/group/flush_spread">
+<img src="flush_spread_title.gif" alt="FLUSH SPREAD" border=0>
+</a>
+
+<!--#include virtual="/includes/header-b" -->
+
+<h2><em>Error Return Codes</em></h2>
+
+Flush Spread uses the same error return codes as Spread. In addition to these 
+<a href="http://www.spread.org/docs/error.html">common error codes</a>, Flush Spread 
+also adds the following error codes:
+
+<pre>
+#define		ILLEGAL_PARAM
+#define		WOULD_BLOCK
+#define		ILLEGAL_MESSAGE_TYPE
+#define		ILLEGAL_STATE
+#define		ILLEGAL_RECEIVERS
+</pre>
+
+<ol>
+<li><tt>ILLEGAL_PARAM:</tt></p>
+
+An illegal parameter was passed to a call. Usually this is the result of specifying a
+negative array size.</p>
+
+<li><tt>WOULD_BLOCK:</tt></p>
+
+A receive call that requested the <tt>DONT_BLOCK</tt> <a href="messages.html">service</a> would
+have blocked.</p>
+
+<li><tt>ILLEGAL_MESSAGE_TYPE:</tt></p>
+
+A multicast call was made with an <a href="misc.html">illegal message type</a>.</p>
+
+<li><tt>ILLEGAL_STATE:</tt></p>
+
+A <a href="FL_multicast.html">FL_multicast</a> or <a href="FL_flush.html">FL_flush</a> call was made
+on a group while that group was in a prohibited state for that call.</p>
+
+<li><tt>ILLEGAL_RECEIVERS:</tt></p>
+
+A subgroup-multicast or unicast call specified receivers that were not currently
+members of the reference group.</p>
+</ol>
+
+Flush Spread also expands the meaning of the following Spread error codes:
+
+<pre>
+#define		ILLEGAL_GROUP
+#define		ILLEGAL_MESSAGE
+</pre>
+
+<ol>
+<li><tt>ILLEGAL_GROUP:</tt></p> 
+
+In addition to representing illegal Spread group names, this error also represents that a
+group-specific call (i.e. - multicast, flush, join, leave) was made illegally while either
+not a member of the group (multicast, flush, leave) or while a member of the group
+(join).</p>
+
+<li><tt>ILLEGAL_MESSAGE:</tt></p>
+
+This error code in Flush Spread has the same meaning as it does in Spread. It means that
+either a send or receive message buffer or scatter was illegal in some manner. This can be
+returned if the size of a buffer is negative or if the number of scat elements is illegal
+(negative or greater than <tt><a href="misc.html">FL_MAX_SCATTER_ELEMENTS</a></tt>).</p>
+
+</ol>
+
+<!--#include virtual="/includes/footer" -->
+
+</body></html>

Added: trunk/docs/flush/docs/flush_spread_title.gif
===================================================================
(Binary files differ)


Property changes on: trunk/docs/flush/docs/flush_spread_title.gif
___________________________________________________________________
Name: svn:mime-type
   + application/octet-stream

Added: trunk/docs/flush/docs/functions.html
===================================================================
--- trunk/docs/flush/docs/functions.html	2005-07-28 14:50:56 UTC (rev 252)
+++ trunk/docs/flush/docs/functions.html	2005-07-31 10:51:40 UTC (rev 253)
@@ -0,0 +1,38 @@
+<html>
+<head><title>Flush Documentation: Flush Spread C Function Interface</title></head>
+
+<body bgcolor="#FFFFFF">
+
+<!--#include virtual="/includes/header-a" -->
+
+<a href="http://www.cnds.jhu.edu/research/group/flush_spread">
+<img src="flush_spread_title.gif" alt="FLUSH SPREAD" border=0>
+</a>
+
+<!--#include virtual="/includes/header-b" -->
+
+<h2><em>Flush Spread C Function Interface</em></h2>
+
+<ul>
+<li><a href="FL_version.html">FL_version</a>
+<li><a href="FL_connect.html">FL_connect</a>
+<li><a href="FL_disconnect.html">FL_disconnect</a>
+<li><a href="FL_join.html">FL_join</a>
+<li><a href="FL_leave.html">FL_leave</a>
+<li><a href="FL_flush.html">FL_flush</a>
+<li><a href="FL_multicast.html">FL_unicast</a>
+<li><a href="FL_multicast.html">FL_scat_unicast</a>
+<li><a href="FL_multicast.html">FL_subgroupcast</a>
+<li><a href="FL_multicast.html">FL_scat_subgroupcast</a>
+<li><a href="FL_multicast.html">FL_multicast</a>
+<li><a href="FL_multicast.html">FL_scat_multicast</a>
+<li><a href="FL_receive.html">FL_receive</a>
+<li><a href="FL_receive.html">FL_scat_receive</a>
+<li><a href="FL_more_msgs.html">FL_more_msgs</a>
+<li><a href="FL_poll.html">FL_poll</a>
+<li><a href="FL_error.html">FL_error</a>
+</ul>
+
+<!--#include virtual="/includes/footer" -->
+
+</body></html>
\ No newline at end of file

Added: trunk/docs/flush/docs/index.html
===================================================================
--- trunk/docs/flush/docs/index.html	2005-07-28 14:50:56 UTC (rev 252)
+++ trunk/docs/flush/docs/index.html	2005-07-31 10:51:40 UTC (rev 253)
@@ -0,0 +1,115 @@
+<html>
+<head><title>Flush Documentation</title></head>
+
+<body bgcolor="#FFFFFF">
+
+<!--#include virtual="/includes/header-a" -->
+
+<a href="http://www.cnds.jhu.edu/research/group/flush_spread">
+<img src="flush_spread_title.gif" alt="FLUSH SPREAD" border=0>
+</a>
+
+<!--#include virtual="/includes/header-b" -->
+
+<h2>Documentation</h2>
+
+Flush Spread is an extension to the <a href="http://www.spread.org/">Spread Wide Area Group
+Communication System</a>.  Flush Spread and Spread are extremely similiar group communication
+systems (GCSs), in the services that they provide, and in their general interface.  Therefore,
+reading Spread's <a href="http://www.spread.org/docs/docspread.html">documentation</a> before
+reading Flush Spread's documentation is highly recommended to get a general grasp of group
+communication and to fill in any holes in this documentation :).</p>
+
+There are two main differences between Spread and Flush Spread:
+(1) Spread provides <a href="http://www.cs.jhu.edu/~yairamir/dcs-94.ps ">extended virtual
+synchrony</a> GCS semantics while Flush Spread provides view synchrony GCS semantics, and (2) Spread
+provides open-group semantics while Flush Spread only provides closed-group semantics.</p>
+
+<h3>Extended Virtual Synchrony (EVS) vs. View Synchrony Semantics (VS)</h3></p>
+
+The main difference between these two models is how they handle view/membership changes.  In an EVS
+system, membership changes occur without a client's intervention, whereas in a VS system a client
+must give its permission before a new view/membership can be installed.  Of course, in "real-life" a
+VS system cannot truly hold-off membership changes (as other clients can crash, network can
+partition, etc.), but what it can do is insulate a client from these changes and not allow new
+members (who either joined or merged) or their messages to be presented to the client without its
+permission.</p>
+
+In a VS system, when an underlying membership change occurs (another member joins/leaves/disconnects,
+network parititions or merges) the GCS generates a flush request message indicating that the current
+view is out-of-date and requesting permission to install a new view.  The client is still allowed to
+send messages at this point, and they may be able to receive messages from surviving members.  When
+the client is ready, it flushes the group, which gives the GCS permission to install a new view.  
+After flushing, a client is not allowed to send any messages to the flushed group until they receive
+the new view of the group.</p>
+
+In a EVS system, when an underlying membership change occurs the GCS figures out the new membership
+and delivers a new view to the client in its normal stream of messages.</p>
+
+In both systems, using the safety guarantees of the different message types and the virtual synchrony
+property (i.e. transitional sets) the client can infer some information about what other members have
+seen without further communication.</p>
+
+Although the difference in these models seems slight, it can lead to quite astonishing differences
+in performance and ease of programming.  In general, the EVS model is faster and scales better with
+respect to membership changes, especially for simple group changes such as joins and leaves.  But
+this performance comes at a price, namely that the client has less control over what is transpiring
+and therefore in general it is harder to program algorithms under EVS than under VS.  For example,
+under EVS the sender doesn't know which other clients might receive its messages until the message
+is delivered back to the sender.  Also, membership changes can come fast and furious and your
+algorithm must be tolerant to every combination of such changes.  Under VS, the client knows exactly
+which other clients <b>might</b> receive its message when it sends -- so the application doesn't
+have to worry about some arbitrary client receiving its messages and mis-interpreting them, for
+example.  Also, membership changes only occur as fast as the applications want them to, although
+several underlying memberships may be collapsed into one.  So they are generally easier to handle
+and track.</p>
+
+<h3>Open-Group vs. Closed-Group Semantics</h3></p>
+
+Open-group semantics allows clients to perform group specific calls, such as multicasts, without
+being a member of the group in question.  Closed-group semantics requires a client to be a full
+fledged member of a group (see below) in order to perform group specific calls.  In general, a
+closed-group GCS is much more picky than an open-group system about a client's state and the
+operations they perform with respect to any particular group.</p>
+
+Under Spread, a client can call group specific calls such as join, leave, and *cast (multicast and
+its variants) pretty much whenever they want regardless of their membership in that group.  Also,
+under Spread the guarantees of multicast messages span differerent groups. For example, if a sender
+sends two FIFO messages to two different groups and a receiver receives both messages, they will
+always receive the first and then the second FIFO message.</p>
+
+Under Flush Spread, a client can make group specific calls such as join, leave, flush, and *cast
+only when in the proper membership state with respect to that group.  For example, leave, flush, and
+*cast calls can only be made when the client is a full-fledged member of the group.  A full-fledged
+member is a client that has joined a group, received a membership including them in the group and is
+not in the process of leaving the group.  Join calls can only be made when the client is not a
+member of the group (either never joined it or whose most recent membership for that group was a
+self-leave) and not already in the process of joining that group.</p>
+
+Also under Flush Spread, *cast and flush calls (i.e. FL_flush) are restricted depending on the group
+membership state.  Flush calls can only be made <b>ONCE</b> per view/membership in response to
+receiving a flush request message.  Once a client flushes a group they are not allowed to *cast to
+that group until they receive the next view/membership for that group.  Flush Spread also does not
+support cross group message semantics.  For example, if a sender sends two FIFO messages to two
+different groups and a receiver receives both messages, they may receive the first and then the
+second <b>OR</b> vice-versa.</p>
+
+<hr>
+
+<h2>C/C++ Documentation</h2>
+
+The following documentation explains the user level C interface to Flush Spread as specified in the
+header file <a href="../src/fl.h">fl.h</a>.  This documentation assumes a knowledge of the ideas,
+discussed above, of view synchronous group communication.
+
+<ol>
+<li><a href="messages.html">Message Types for Data and Membership Messages</a>
+<li><a href="access.html">Message Type Access Functions</a>
+<li><a href="error.html">Error Return Codes</a>
+<li><a href="misc.html">Miscellaneous Constants</a>
+<li><a href="functions.html">Flush Spread C Function Interface</a>
+</ol>
+
+<!--#include virtual="/includes/footer" -->
+
+</body></html>

Added: trunk/docs/flush/docs/messages.html
===================================================================
--- trunk/docs/flush/docs/messages.html	2005-07-28 14:50:56 UTC (rev 252)
+++ trunk/docs/flush/docs/messages.html	2005-07-31 10:51:40 UTC (rev 253)
@@ -0,0 +1,61 @@
+<html>
+<head><title>Flush Documentation: Message Types for Data and Membership Messages</title></head>
+
+<body bgcolor="#FFFFFF">
+
+<!--#include virtual="/includes/header-a" -->
+
+<a href="http://www.cnds.jhu.edu/research/group/flush_spread">
+<img src="flush_spread_title.gif" alt="FLUSH SPREAD" border=0>
+</a>
+
+<!--#include virtual="/includes/header-b" -->
+
+<h2><em>Message Types for Data and Membership Messages</em></h2>
+
+Flush Spread uses the same service and membership types as Spread. In addition to these
+<a href="http://www.spread.org/docs/messages.html">common types</a>, Flush Spread also
+adds the following service types:
+
+<pre>
+#define 	DONT_BLOCK
+#define		FLUSH_REQ_MESS
+#define		SUBGROUP_CAST
+</pre>
+
+<ol> 
+
+<li><tt>DONT_BLOCK:</tt></p>
+
+This service type can be requested when making a receive call. If the receive call would block while
+receiving on the mailbox, the call will fail and return the error code <tt>WOULD_BLOCK</tt>.
+Currently, Spread client libraries (v3.14 and earlier) do not support non-blocking receives. This
+means that even if the caller sets the <tt>DONT_BLOCK</tt> service flag a call to receive may take
+longer than would normally be expected of a non-blocking I/O call. However, the call should never
+block permanently.</p>
+
+<li><tt>FLUSH_REQ_MESS:</tt></p>
+
+This message type can be set upon returning from a receive call. It indicates that the received
+message was a flush request message for the group contained in sender. A flush request message is
+generated by Flush Spread when the underlying membership of that group changes. To install the new
+membership and make progress, the receiver must eventually respond to this signal by flushing that
+group by calling <a href="FL_flush.html">FL_flush</a>. Receiving a flush request message is an
+important part of the view synchrony GCS semantics and has pretty drastic importance. See <a
+href="FL_receive.html">FL_receive</a> for a full description of the restrictions incurred by and
+ramifications of receiving a flush request. Note that a flush request is neither a membership message
+nor a regular message. See the <a href="access.html">message type access functions</a> for a macro to
+test if a message is a flush request message.</p>
+
+<li><tt>SUBGROUP_CAST:</tt></p>
+
+This message type can be set upon returning from a receive call. It indicates that the
+received message was only multicasted to a subset of a group. This message was only sent
+to the members in the groups array in the range [0, *num_groups-1). The group that this
+subgroup-multicast occurred in is contained in groups[*num_groups-1].</p>
+
+</ol>
+
+<!--#include virtual="/includes/footer" -->
+
+</body></html>

Added: trunk/docs/flush/docs/misc.html
===================================================================
--- trunk/docs/flush/docs/misc.html	2005-07-28 14:50:56 UTC (rev 252)
+++ trunk/docs/flush/docs/misc.html	2005-07-31 10:51:40 UTC (rev 253)
@@ -0,0 +1,33 @@
+<html>
+<head><title>Flush Documentation: Miscellaneous Constants</title></head>
+
+<body bgcolor="#FFFFFF">
+
+<!--#include virtual="/includes/header-a" -->
+
+<a href="http://www.cnds.jhu.edu/research/group/flush_spread">
+<img src="flush_spread_title.gif" alt="FLUSH SPREAD" border=0>
+</a>
+
+<!--#include virtual="/includes/header-b" -->
+
+<h2><em>Miscellaneous Constants</em></h2>
+
+<pre>
+#define		FL_MAX_SCATTER_ELEMENTS
+</pre>
+
+Flush Spread restricts the maximum number of scatter elements a caller is allowed to use
+in multicast calls to be less than or equal to <tt>FL_MAX_SCATTER_ELEMENTS</tt>. See Spread's
+<tt><a href="http://www.spread.org/docs/types.html">MAX_SCATTER_ELEMENTS</a></tt>.</p>
+
+<pre>
+#define		FL_MIN_LEGAL_MESS_TYPE
+</pre>
+
+Flush Spread restricts the range of message types (int16) a caller is allowed to use in
+multicast calls to be greater than or equal to <tt>FL_MIN_LEGAL_MESS_TYPE</tt>.</p>
+
+<!--#include virtual="/includes/footer" -->
+
+</body></html>

Added: trunk/docs/flush/index.html
===================================================================
--- trunk/docs/flush/index.html	2005-07-28 14:50:56 UTC (rev 252)
+++ trunk/docs/flush/index.html	2005-07-31 10:51:40 UTC (rev 253)
@@ -0,0 +1,56 @@
+<html>
+<head><title>The Group Communication Project</title></head>
+
+<body bgcolor="#FFFFFF">
+
+<!--#include virtual="/includes/header-a" -->
+
+<img src="docs/flush_spread_title.gif" alt="FLUSH SPREAD">
+
+<!--#include virtual="/includes/header-b" -->
+
+</p>
+<font size= -1>
+<DD><A href="http://www.cs.jhu.edu/~yairamir">Yair Amir</A></DD>
+<DD><A href="http://www.cnds.jhu.edu/~jschultz">John Schultz</A></DD>
+<DD><A href="http://www.cnds.jhu.edu/~jonathan">Jonathan Stanton</A></DD>
+</font>
+<HR>
+</p>
+
+<p>
+The Flush Spread project focuses on providing view synchrony group communication
+semantics built on top of Spread's extended virtual synchrony semantics. Currently, these
+semantics are provided by a client library that links with Spread's client library.
+</p>
+
+<p>
+Obviously you need Spread for Flush Spread to work ... so go check out
+<a href="http://www.spread.org">Spread</a>.
+</p>
+
+<p>
+Flush Spread includes a distribution of the Stdutil library, which is a toolkit of
+high-performance C data structures and utility functions. If you are interested, you
+can learn more about it <a href="http://www.cnds.jhu.edu/software/stdutil">here</a>.
+</p>
+
+<H3>Flush Spread Resources:</H3>
+
+<UL>
+<li>Read Flush Spread's <a href="README">README</a> file.
+
+<li>Read the Flush Spread <a href="FLUSH_LICENSE">license</a> or the Stdutil 
+    <a href="stdutil/STDUTIL_LICENSE">license</a>.<br>
+
+<li>Read the Flush Spread <a href="docs/index.html">documentation</a>.<br>
+
+<li><a href="http://www.cnds.jhu.edu/download/flush_spread.tar.gz">Download</a> Flush Spread now!<br>
+</UL>
+
+E-mail <a href="mailto:flush_spread at cnds.jhu.edu"><i>flush_spread at cnds.jhu.edu</i></a>
+for more information.</p>
+
+<!--#include virtual="/includes/footer" -->
+
+</body></html>

Added: trunk/docs/flush/man/FL_connect.3
===================================================================
--- trunk/docs/flush/man/FL_connect.3	2005-07-28 14:50:56 UTC (rev 252)
+++ trunk/docs/flush/man/FL_connect.3	2005-07-31 10:51:40 UTC (rev 253)
@@ -0,0 +1,105 @@
+.TH FL_connect 3 "Dec 2000" "Flush Spread" "User Manuals"
+.SH NAME
+FL_connect \- connect an application to a Spread daemon using Flush Spread semantics.
+.SH SYNOPSIS
+.B #include <fl.h>
+
+.BI "int FL_connect(const char *" daemon_name ", const char *" user_name ", int " priority ", int " group_membership ", mailbox *" mbox ", char *" private_name );
+.SH DESCRIPTION
+.B FL_connect
+is the initial call an application must make to establish a
+connection with a Spread daemon.  All other Flush Spread calls
+refer to a valid 
+.I mbox 
+returned by this function.
+
+The 
+.I daemon_name
+is the name of the Spread daemon with which to connect.  It should be a
+string in one of the following forms:
+.RS
+.TP
+.B "4803" 
+This will connect to the Spread daemon on the local 
+machine running on port 4803.  This form cannot be
+used to connect to a Windows95/NT machine.
+.TP
+.B "4803 at localhost" 
+This will also connect to the Spread daemon 
+on the local machine running on port 4803.
+This form can be used on Windows95/NT machines.
+.TP
+.BR "4803 at host.domain.edu " or " 4803 at 128.220.221.99"     
+This will connect to the machine identified by either 
+domain name or IP address at the specified port. 
+.RE
+
+The
+.I user_name 
+is the name this connection would like to be known as.  It must be
+unique on the machine running the spread daemon.  The name can be an
+arbitrary length string with the same character restrictions as a
+group name (mainly it cannot contain the '#' character).
+
+.I priority
+is a boolean (non-zero true, zero false) for whether this connection
+will be a "priority" connection or not. Currently this has no effect.
+
+.I group_membership
+is a boolean for whether this connection will receive group membership
+messages or not.  Usually for Flush Spread applications this parameter
+should be true.  If your application doesn't need group membership
+messages, then Spread may provide the semantics that you need more
+efficiently than Flush Spread.
+
+The
+.I mbox
+should be a pointer to a mailbox variable.  After the 
+.B FL_connect
+call successfully returns, this variable will hold the valid mbox for
+this new connection.
+
+The
+.I private_name
+should be a pointer to a string big enough to hold at least
+MAX_GROUP_NAME characters.  After the call returns it will contain the
+private group name of this connection.  These group names are what are
+reported in membership messages and can be used to send unicast and
+subgroup-multicast messages to this connection.  No other applications
+can join this special group.
+
+.SH "RETURN VALUES"
+Returns 
+.B ACCEPT_SESSION 
+on success or one of the following errors ( < 0 ):
+.TP
+.B ILLEGAL_SPREAD
+The 
+.I daemon_name
+given to connect to was illegal for some reason. Usually because
+it was a unix socket on Windows95/NT, an improper format for a host
+or an illegal port number
+.TP
+.B COULD_NOT_CONNECT
+Lower level socket calls failed to allow a connection to the 
+specified spread daemon right now.
+.TP
+.B CONNECTION_CLOSED
+During communication to establish the connection errors occured
+and the setup could not be completed.
+.TP
+.B REJECT_VERSION
+The daemon and/or libraries have a version mismatch.
+.TP
+.B REJECT_NO_NAME
+No user name was provided.
+.TP
+.B REJECT_ILLEGAL_NAME
+Name provided violated some requirement (length or used an illegal character)
+.TP
+.B REJECT_NOT_UNIQUE
+Name provided is not unique on this daemon. Recommended response is to try
+again with a different name.
+.SH AUTHOR
+John Schultz <jschultz at cnds.jhu.edu>
+

Added: trunk/docs/flush/man/FL_disconnect.3
===================================================================
--- trunk/docs/flush/man/FL_disconnect.3	2005-07-28 14:50:56 UTC (rev 252)
+++ trunk/docs/flush/man/FL_disconnect.3	2005-07-31 10:51:40 UTC (rev 253)
@@ -0,0 +1,26 @@
+.TH FL_disconnect 3 "Dec 2000" "Flush Spread" "User Manuals"
+.SH NAME
+FL_disconnect \- destroy a Flush Spread connection between an application and a Spread daemon.
+.SH SYNOPSIS
+.B #include <fl.h>
+
+.BI "int FL_disconnect(mailbox " mbox ");"
+.SH DESCRIPTION
+.B FL_disconnect
+should be called when the application is finished
+with a connection to a Spread daemon.  The application may have
+other connections still open to the daemon and may open a new
+connection after disconnecting. Any data that was available on
+the connection will be lost after the call returns.
+
+The 
+.I mbox 
+should be for the connection you wish to close.
+.SH "RETURN VALUES"
+Returns 0 on success or 
+.B ILLEGAL_SESSION
+when the
+.I mbox
+given does not represent a valid connection.
+.SH AUTHOR
+John Schultz <jschultz at cnds.jhu.edu>

Added: trunk/docs/flush/man/FL_error.3
===================================================================
--- trunk/docs/flush/man/FL_error.3	2005-07-28 14:50:56 UTC (rev 252)
+++ trunk/docs/flush/man/FL_error.3	2005-07-31 10:51:40 UTC (rev 253)
@@ -0,0 +1,13 @@
+.TH FL_error 3 "Dec 2000" "Flush Spread" "User Manuals"
+.SH NAME
+FL_error \- Flush Spread error string reporting
+.SH SYNOPSIS
+.B #include <fl.h>
+
+.BI "void FL_error(int " error_code ");"
+.SH DESCRIPTION
+.B FL_error
+prints an error string based on the Flush Spread or Spread error code
+.IR error_code .
+.SH AUTHOR
+John Schultz <jschultz at cnds.jhu.edu>

Added: trunk/docs/flush/man/FL_flush.3
===================================================================
--- trunk/docs/flush/man/FL_flush.3	2005-07-28 14:50:56 UTC (rev 252)
+++ trunk/docs/flush/man/FL_flush.3	2005-07-31 10:51:40 UTC (rev 253)
@@ -0,0 +1,94 @@
+.TH FL_flush 3 "Dec 2000" "Flush Spread" "User Manuals"
+.SH NAME
+FL_flush \- flush a group in response to receiving a flush request message.
+.SH SYNOPSIS
+.B #include <fl.h>
+
+.BI "int FL_flush(mailbox " mbox ", const char *" group_name ");"
+.SH DESCRIPTION
+.B FL_flush
+flushes the group named
+.I group_name
+in which the connection represented by
+.I mbox
+has received a flush request message.  A flush request message is
+delivered in a group when the underlying membership of that group
+changes; it is a signal to the application that the membership has
+changed and some of the original members may have "gone away" (meaning
+that they are no longer in the group with this connection).  However,
+Flush Spread will not install a new membership in the group until each
+of the members of the new membership flush the group.  Flushing a
+group gives Flush Spread the permission of this connection to go ahead
+and install a new view/membership for the group.
+
+During the time after receiving a flush request message but before
+flushing the group the application must be _VERY_ careful about
+calling receive functions; if the application isn't careful enough it
+can permantently block the entire group.  For example, say the
+underlying membership of a group of 10 members changed such that this
+connection is now alone in the group (there is no way an application
+can detect the make-up of a group after receiving a flush request).
+If the application simply calls receive before flushing the group and
+hasn't ensured that it will receive a message somehow (see below) it
+will permanently block itself in the receive and any other members
+that might later be added to that group.
+
+There are a couple ways to avoid this problem, which isn't a bug by
+the way :P, (1) don't call receive functions after receiving a flush
+request message and before flushing the group, (2) call receive but
+set the DONT_BLOCK service flag; if there aren't any messages to
+receive it will break out with a WOULD_BLOCK error, (3) ensure that
+there will be a message to receive somehow.  There are several ways to
+do this, but one 100% sure way to do this is if the application sends
+or has sent a message on this connection (with the SELF_DISCARD
+service _NOT_ set) that it has not yet received back.  In this case,
+the application will eventually receive its own message (even if it
+was to another group) on the connection and can therefore be assured
+of not blocking permanently.
+
+The application _IS_ allowed to send messages to a group after
+receiving a flush request for that group and before flushing the
+group.  However, only a subset of the original members of the current
+view will receive these messages (note that I said a subset, not a
+strict subset).  Technically, this is always the case in Spread: an
+application can determine which other members received certain
+messages for sure by (1) application level message receipt
+acknowledgement, (2) employing the safety properties of SAFE messages
+(although this doesn't guarantee that those members actually processed
+and handled the message), (3) employing the virtual synchrony property
+and the transitional set of the new membership when it is installed
+(although, again, this doesn't guarantee that those members actually
+processed and handled the messages). See FL_receive or read up on
+group communication for more in-depth discussions of this matter.
+
+Anyways, when the application is ready it can flush the group.  Once
+it flushes the group it is not allowed to send any messages to that
+group, until it receives the new membership for that group.  If the
+application breaks this rule, it will receive an ILLEGAL_STATE error.
+
+.SH "RETURN VALUES"
+Returns 0 on success or one of the following errors ( < 0 ):
+.TP
+.B ILLEGAL_SESSION
+The connection represented by 
+.I mbox
+is illegal. Usually because it is not active.
+.TP
+.B ILLEGAL_GROUP
+The 
+.I group_name
+given to flush was illegal for some reason, usually because it was of
+length 0 or length > MAX_GROUP_NAME.  This error can also be returned
+when a group is flushed for a connection that it has not yet joined or
+is already leaving.
+.TP
+.B ILLEGAL_STATE
+A connection may flush a group only once in response to each flush
+request message delivered in that group.  If you violate this rule you
+will get an ILLEGAL_STATE error.
+.TP
+.B CONNECTION_CLOSED
+Errors occurred during communication and the flush could not be
+initiated.
+.SH AUTHOR
+John Schultz <jschultz at cnds.jhu.edu>

Added: trunk/docs/flush/man/FL_join.3
===================================================================
--- trunk/docs/flush/man/FL_join.3	2005-07-28 14:50:56 UTC (rev 252)
+++ trunk/docs/flush/man/FL_join.3	2005-07-31 10:51:40 UTC (rev 253)
@@ -0,0 +1,38 @@
+.TH FL_join 3 "Dec 2000" "Flush Spread" "User Manuals"
+.SH NAME
+FL_join \- join a connection to a Flush Spread group
+.SH SYNOPSIS
+.B #include <fl.h>
+
+.BI "int FL_join(mailbox " mbox ", const char *" group_name ");"
+.SH DESCRIPTION
+.B FL_join
+joins the connection represented by 
+.I mbox
+to a group with the name
+.IR group_name .
+If the group does not exist among the Spread daemons it is created,
+otherwise the connection is joined to the existing group.  A
+connection may not join a group it is already a member of, which it
+has already joined, or from which it is currently leaving.
+.SH "RETURN VALUES"
+Returns 0 on success or one of the following errors ( < 0 ):
+.TP
+.B ILLEGAL_GROUP
+The 
+.I group_name
+given to join was illegal for some reason, usually because it was of
+length 0 or length > MAX_GROUP_NAME.  This error is also returned if a
+group with which this connection is already involved (i.e. - already
+joining, already joined, currently leaving) is joined again.
+.TP
+.B ILLEGAL_SESSION
+The connection represented by
+.I mbox
+is illegal. Usually because it is not active.
+.TP
+.B CONNECTION_CLOSED
+Errors occurred during communication and the join could not be
+initiated.
+.SH AUTHOR
+John Schultz <jschultz at cnds.jhu.edu>

Added: trunk/docs/flush/man/FL_leave.3
===================================================================
--- trunk/docs/flush/man/FL_leave.3	2005-07-28 14:50:56 UTC (rev 252)
+++ trunk/docs/flush/man/FL_leave.3	2005-07-31 10:51:40 UTC (rev 253)
@@ -0,0 +1,40 @@
+.TH FL_leave 3 "Dec 2000" "Flush Spread" "User Manuals"
+.SH NAME
+FL_leave \- remove a connection from a Flush Spread group
+.SH SYNOPSIS
+.B #include <fl.h>
+
+.BI "int FL_leave(mailbox " mbox ", const char *" group_name ");"
+.SH DESCRIPTION
+.B FL_leave
+removes the connection represented by
+.I mbox
+from a Flush Spread group with the name
+.IR group_name .
+A connection may only leave a group after it has been installed as a
+member of the group by being included in a Flush Spread
+view/membership.  Also a connection may not leave a group if it has
+already requested to leave that group and the self-leave message has
+not yet been received.
+.SH "RETURN VALUES"
+Returns 0 on success or one of the following errors ( < 0 ):
+.TP
+.B ILLEGAL_GROUP
+The 
+.I group_name
+given to leave was illegal for some reason. Usually because it was of
+length 0 or length > MAX_GROUP_NAME.  This error is also returned if
+the connection was not a full-fledged member (i.e. - not yet included
+in a Flush membership message for that group, or already leaving) of
+the group.
+.TP
+.B ILLEGAL_SESSION
+The connection represented by 
+.I mbox
+is illegal. Usually because it is not active.
+.TP
+.B CONNECTION_CLOSED
+Errors occurred during communication
+and the leave could not be initiated.
+.SH AUTHOR
+John Schultz <jschultz at cnds.jhu.edu>

Added: trunk/docs/flush/man/FL_more_msgs.3
===================================================================
--- trunk/docs/flush/man/FL_more_msgs.3	2005-07-28 14:50:56 UTC (rev 252)
+++ trunk/docs/flush/man/FL_more_msgs.3	2005-07-31 10:51:40 UTC (rev 253)
@@ -0,0 +1,41 @@
+.TH FL_more_msgs 3 "Dec 2000" "Flush Spread" "User Manuals"
+.SH NAME
+FL_more_msgs \- returns the number of complete messages buffered and
+ready to be received on a connection.
+.SH SYNOPSIS
+.B #include <fl.h>
+
+.BI "int FL_more_msgs(mailbox " mbox ");"
+.SH DESCRIPTION
+.B FL_more_msgs
+allows an application to check to see if any complete messages are
+already buffered and ready to be received on the connection represented by
+.IR mbox .
+
+.BR NOTE , 
+that this function 
+.B CANNOT 
+be used as an I/O polling function to check if a receive call should
+be done.  If this function returns zero there still might be a message
+ready to receive on the connection: FL_poll, and file descriptor
+selects/polls can detect if there is activity (not necessarily a
+message) on a connection, while DONT_BLOCK receive semantics can
+detect whether or not there is a message on the connection and
+FL_more_msgs can only answer if there are any buffered messages
+already on the connection.
+
+This function is merely a helper function to re-get the current status
+of the
+.I more_messes
+parameter returned from a call to 
+.BR FL_receive .
+.SH "RETURN VALUES"
+Returns the number of complete buffered messages ready to be received,
+or one of the following errors ( < 0):
+.TP
+.B ILLEGAL_SESSION
+The connection represented by 
+.I mbox
+is illegal, usually because it is not active.
+.SH AUTHOR
+John Schultz <jschultz at cnds.jhu.edu>

Added: trunk/docs/flush/man/FL_multicast.3
===================================================================
--- trunk/docs/flush/man/FL_multicast.3	2005-07-28 14:50:56 UTC (rev 252)
+++ trunk/docs/flush/man/FL_multicast.3	2005-07-31 10:51:40 UTC (rev 253)
@@ -0,0 +1,162 @@
+.TH FL_multicast 3 "Dec 2000" "Flush Spread" "User Manuals"
+.SH NAME
+FL_unicast, FL_scat_unicast, FL_subgroupcast, FL_scat_subgroupcast, FL_multicast, FL_scat_multicast \- Multicast messages to subsets of Flush Spread groups.
+.SH SYNOPSIS
+.B #include <fl.h>
+
+.BI "int FL_unicast(mailbox " mbox ", service " serv_type ", const char *" group_name ", const char *" recvr_name ", int16 " mess_type ", int " mess_len ", const char *" mess ");"
+
+.BI "int FL_scat_unicast(mailbox " mbox ", service " serv_type ", const char *" group_name ", const char *" recvr_name ", int16 " mess_type ", const scatter *" scat ");"
+
+.BI "int FL_subgroupcast(mailbox " mbox ", service " serv_type ", const char *" group_name ", int " num_recvrs ", const char " recvr_names "[][MAX_GROUP_NAME], int16 " mess_type ", int " mess_len ", const char *" mess ");"
+
+.BI "int FL_scat_subgroupcast(mailbox " mbox ", service " serv_type ", const char *" group_name ", int " num_recvrs ", const char " recvr_names "[][MAX_GROUP_NAME], int16 " mess_type ", const char *" scat ");"
+
+.BI "int FL_unicast(mailbox " mbox ", service " serv_type ", const char *" group_name ", int16 " mess_type ", int " mess_len ", const char *" mess ");"
+
+.BI "int FL_scat_unicast(mailbox " mbox ", service " serv_type ", const char *" group_name ", int16 " mess_type ", const scatter *" scat ");"
+.SH DESCRIPTION
+.B FL_multicast
+and its variants all multicast a message from the connection represented by
+.I mbox
+to a subset of the membership of the Flush Spread group named
+.IR group_name .
+
+Note, that unlike Spread, Flush Spread does not provide guarantees
+about messages sent to different groups; meaning that FIFO, CAUSAL,
+SAFE, etc. safety properties are not maintained across different Flush
+Spread groups, but only within a group to which they are sent. For
+example, if an application sends a FIFO message A to the private group
+G and then sends another FIFO message B to the regular group H, of
+which G is a member, then G could receive B before A or vice versa.
+
+In order to multicast a message to a group, the destination group 
+.I group_name
+must either be a private group name, or this connection must be a
+full-fledged member of that regular group (has joined and been
+included in a membership of the group, and is not leaving the group,
+see FL_join, FL_leave) and not be in an illegal state to multicast (by
+having recently flushed the group and not yet received its next
+view/membership, see FL_flush).
+
+The
+.I serv_type
+is a type field that should be set to the service type this message
+requires. The valid flags for multicasting messages are:
+
+.RS
+.B UNRELIABLE_MESS
+.br
+.TB
+.B RELIABLE_MESS 
+.br
+.TB
+.B FIFO_MESS
+.br
+.TB
+.B CAUSAL_MESS
+.br
+.TB
+.B AGREED_MESS
+.br
+.TB
+.B SAFE_MESS
+.RE
+
+This type field can be bit-wise ORed with other flags like
+SELF_DISCARD if desired.  Currently SELF_DISCARD is the only
+additional flag for multicasts: SELF_DISCARD will cause the
+multicasted message to NOT be delivered back to this connection.
+
+The string
+.I group_name
+indicates the name of the Flush Spread group to which this message
+should be sent.  This group can be either a regular or a private Flush
+Spread group, as dicussed above.  The multicast variants sends the
+message to all members of a group, while the unicast and subgroupcast
+variants allow the sender to specify a subset of the group to which to
+send the message.  For the unicast variants the sender specifies the
+private group name of the intended recipient in the variable
+.IR recvr_name .  
+For the subgroupcast variants the sender specifies the number of
+recipients in the variable
+.I num_recvrs
+and a doubly scripted array of the members' private group names in the
+variable
+.IR recvr_names ,
+where duplicate names are tolerated.  For the unicast and subgroupcast
+variants all specified receivers must have been members of the most
+recently installed view/membership of the group.  If the application
+breaks this rule it will receive an ILLEGAL_RECEIVERS error.
+
+The
+.I mess_type
+is a 16 bit integer which can be used by the application arbitrarily.
+However, Flush Spread restricts which values can be used to be greater
+than or equal to FL_MIN_LEGAL_MESS_TYPE.  If the application sends
+with a message type of less than FL_MIN_LEGAL_MESS_TYPE, then the
+error ILLEGAL_MESSAGE_TYPE will be returned.  The intent of this
+message type is that the application can use it to name different
+kinds of data messages so they can be differentiated without looking
+into the body of the message.  This value
+.B WILL
+be endian corrected upon returning. 
+
+The non-scatter variants use a single buffer to pass the body of the
+message to be sent.  The
+.I mess_len
+field gives the message length in bytes. While the
+.I mess
+field is a pointer to the buffer containing the message.  For the
+scatter variants, both of these parameters are replaced with one
+pointer,
+.IR scat_mess ,
+to a scatter structure, which is similiar to an iovec.  This allows
+messages made up of several parts to be sent without an extra copy on
+systems which support scatter-gather.  However, the number of scatter
+elements is restricted to be non-negative and less than or equal to
+FL_MAX_SCATTER_ELEMENTS.  If a buffer length is negative or an illegal
+number of scatter elements is used then ILLEGAL_MESSAGE will be
+returned.
+.SH "RETURN VALUES"
+Returns the number of bytes sent on success or one of the following
+errors ( < 0 ):
+.TP
+.B ILLEGAL_SESSION
+The connection represented by 
+.I mbox
+was illegal, usually because it is no longer active.
+.TP
+.B ILLEGAL_GROUP
+The 
+.I group_name
+given to multicast to was illegal for some reason. Usually because it
+was of length 0 or length > MAX_GROUP_NAME.  This error can also be
+returned if the connection was not a full-fledged member (i.e. - not
+yet included in a Flush membership message for the group, or already
+leaving) of a regular group.
+.TP
+.B ILLEGAL_RECEIVERS
+A unicast or subgroupcast specified private group names that weren't
+members of
+.IR group_name 's
+most recent view/membership.
+.TP
+.B ILLEGAL_STATE
+A multicast to a group was attempted after flushing the group and
+before receiving the next view/membership for that group.
+.TP
+.B ILLEGAL_SERVICE
+If the service type was using bits reserved by Flush Spread or Spread.
+.TP
+.B ILLEGAL_PARAM
+An illegal parameter, such as a negative array size was passed.
+.TP
+.B ILLEGAL_MESSAGE
+The message had an illegal structure, like a negative buffer size or
+an illegal number of scatter elements.
+.TP
+.B CONNECTION_CLOSED
+Errors occurred during communication and the multicast could not be completed.
+.SH AUTHOR
+John Schultz <jschultz at cnds.jhu.edu>

Added: trunk/docs/flush/man/FL_poll.3
===================================================================
--- trunk/docs/flush/man/FL_poll.3	2005-07-28 14:50:56 UTC (rev 252)
+++ trunk/docs/flush/man/FL_poll.3	2005-07-31 10:51:40 UTC (rev 253)
@@ -0,0 +1,32 @@
+.TH FL_poll 3 "Dec 2000" "Flush Spread" "User Manuals"
+.SH NAME
+FL_poll \- returns the amount, in bytes, of activity on a connection.
+.SH SYNOPSIS
+.B #include <fl.h>
+
+.BI "int FL_poll(mailbox " mbox ");"
+.SH DESCRIPTION
+.B FL_poll
+is a way to poll the connection represented by
+.I mbox
+to see if there is any activity on that connection.  
+
+.BR NOTE ,
+however that activity does
+.B NOT
+necessarily mean that a message is available to be read, as in Spread.
+Only a call to FL_more_msgs that returns a positive number of buffered
+messages or a DONT_BLOCK receive call can check for sure to see if a
+message is now available to be read.  If you don't use either of these
+semantics, then you may enter into a blocking receive call, which can
+sometimes be dangerous (see FL_flush).
+.SH "RETURN VALUES"
+Returns the number of bytes of activity currently on the connection,
+or one of the following errors ( < 0 ):
+.TP
+.B ILLEGAL_SESSION
+The connection represented by 
+.I mbox
+is illegal, usually because it is not active.
+.SH AUTHOR
+John Schultz <jschultz at cnds.jhu.edu>

Added: trunk/docs/flush/man/FL_receive.3
===================================================================
--- trunk/docs/flush/man/FL_receive.3	2005-07-28 14:50:56 UTC (rev 252)
+++ trunk/docs/flush/man/FL_receive.3	2005-07-31 10:51:40 UTC (rev 253)
@@ -0,0 +1,388 @@
+.TH FL_receive 3 "Dec 2000" "Flush Spread" "User Manuals"
+.SH NAME
+Fl_receive, FL_scat_receive \- receive a message on a Flush Spread connection.
+.SH SYNOPSIS
+.B #include <fl.h>
+
+.BI "int FL_receive(mailbox " mbox ", service *" serv_type ", char " sender "[MAX_GROUP_NAME], int " max_groups ", int *" num_groups ", char " groups "[][MAX_GROUP_NAME], int16 *" mess_type ", int *" endian_mismatch ", int " max_mess_len ", char *" mess ", int *" more_messes ");"
+
+.BI "int FL_scat_receive(mailbox " mbox ", service *" serv_type ", char " sender "[MAX_GROUP_NAME], int " max_groups ", int *" num_groups ", char " groups "[][MAX_GROUP_NAME], int16 *" mess_type ", int *" endian_mismatch ", scatter *" scat_mess ", int *" more_messes ");"
+.SH DESCRIPTION
+.B FL_receive
+is the general purpose message receipt function for Flush Spread.
+Messages for all groups joined on this connection will arrive on the
+same mailbox.  A call to
+.B FL_receive
+will perform a receive on a connection to get a single message from
+any one of the groups to which it is joined.  
+
+After a call to receive completes, a number of the passed fields are
+set to values indicating meta-information about the message (such as
+destination group, message type, endianness, etc).  The meanings of
+these different meta fields depends on the type of message received
+and the receipt services requested.
+
+Input Parameters:
+.RS
+.TP 1.1i
+.I mbox
+Represents the connection on which to receive a message.
+.TP 1.1i
+.I serv_type
+A pointer to a service bit field, requesting some receive services.
+This field can have the DONT_BLOCK and/or DROP_RECV service bit flags
+flipped on.  This bit field should be zeroed out before each call if
+no special receive service is to be requested.
+.TP 1.1i
+.I sender
+A pointer to a character array with storage for at least
+MAX_GROUP_NAME characters.
+.TP 1.1i
+.I max_groups
+A non-negative int representing how many group names the application
+is willing to receive in this receive call.
+.TP 1.1i
+.I num_groups
+A pointer to an int.
+.TP 1.1i
+.I groups
+An array of group names containing storage for at least 
+.I max_groups
+* MAX_GROUP_NAME characters.
+.TP 1.1i
+.I mess_type
+A pointer to an int16.
+.TP 1.1i
+.I endian_mismatch
+A pointer to an int.
+.TP 1.1i
+.I max_mess_len
+A non-negative integer representing how many bytes of data the
+application is willing to receive in this call.
+.TP 1.1i
+.I mess
+A pointer to a buffer containing storage for at least
+.I max_mess_len
+bytes.
+.TP 1.1i
+.I scat_mess
+A pointer to a scatter (a cousin of an iovec) with a non-negative
+number of scatter elements that is less than or equal to
+MAX_SCATTER_ELEMENTS (that's not a typo :), and all non-negative
+buffer lengths in the indicated scatter elements to get scatter-gather
+semantics.
+.RE
+
+Receive Semantics:
+
+Normally, a call to receive will block if no messages are immediately
+available.
+
+Normally, when calling a receive function if a user's buffer
+.RI ( groups , 
+.IR mess ,
+or 
+.IR scat_mess ) 
+is too small to contain the data to be returned, then a
+GROUPS_TOO_SHORT or BUFFER_TOO_SHORT error code will be returned.  In
+this case, the
+.IR serv_type ,
+and 
+.IR mess_type
+of the message are filled in from the offending message and the
+parameters
+.IR num_groups ,
+and 
+.I endian_mismatch
+reflect information about the necessary buffers' sizes.  If
+.I max_groups
+was big enough, then
+.RI * num_groups
+will be zero, otherwise it will be the negative of how many group
+names are available to be received (i.e. -7 means the 
+.I max_groups
+was less than 7 and 7 group names can be received).  If
+.I max_mess_len
+or
+.I scat_mess
+was big enough then
+.RI * endian_mismatch
+will be zero, otherwise it will be the negative of how much data is
+available to be received (i.e. -32768 means the msg buffer was too
+small and 32768 bytes of data can be received).  The offending message
+is still available to be received through later calls to receive with
+appropriately sized user buffers.
+
+The parameter
+.I serv_type
+allows the applications to request different receive services that
+affect the normal semantics of receive.  Currently, the only services
+are DONT_BLOCK and DROP_RECV.
+
+The DONT_BLOCK service makes a receive call non-blocking.  With this
+service, the receive call will return quickly either with a message or
+with the error code WOULD_BLOCK if no message is available.  Using
+this service is the only way to detect if a message is ready on the
+connection in a non-blocking manner in Flush Spread.
+
+The DROP_RECV service forces Flush Spread to read the current message
+off of the connection regardless of whether or not the user's buffers
+are big enough to fit all of the data.  In this case, a TOO_SHORT
+error code is still returned even though the call actually
+succeeded. Anyway, as much data as can be fit into the user's buffers
+will be stuffed into them, and that message will no longer be on the
+connection.  Also,
+.RI * num_groups
+will still be set to the negative size of how many names were
+available (if 
+.I groups
+wasn't big enough), but
+.RI * endian_mismatch
+will not reflect the size of the data that could have been received (if
+.I mess
+or
+.I scat_mess
+wasn't big enough), instead it indicates whether or not the sending
+machine had an opposite endianness or not.  Using DROP_RECV in this
+manner, when a buffer problem occurs there is no way to determine how
+big the data portion of the message actually was.  This service is
+meant to be used when a call to receive without DROP_RECV fails with a
+buffer error, but it is determined that the message isn't all that
+important, or something along those lines; otherwise reallocate your
+receive buffers appropriately and re-call receive again without
+the DROP_RECV service.
+
+Output Parameters:
+
+Upon a successful return from receive, 
+.RI * serv_type 
+will be filled out with the message type that was just received.  The
+specific type of message received can be tested using the different
+message and membership type access macros.  The rest of the
+parameters' meanings differ depending on the
+.IR serv_type . 
+
+Regular Messages:
+
+If the
+.I serv_type
+is a REG_MESSAGE (i.e. a data message) then:
+
+The parameter
+.I sender
+will be filled with the private group name of the sending connection.
+
+The parameter
+.I num_groups
+will be set to the number of group names filled into
+.IR groups .
+
+The 
+.I groups
+array will contain the group to which this message was sent.  If the
+message is a SUBGROUP_CAST (check the service type, see FL_multicast)
+then all of the recipients' private group names will be listed and the
+last name in the array (i.e.
+.RI groups[* num_groups -1]) 
+will be the regular group to which this message was sent.
+
+If DROP_RECV is being used, and 
+.I groups
+is too small, then as many names as can fit will be filled in as above
+described.  If the message was a SUBGROUP_CAST, then the last group in
+the array (i.e.
+.RI groups[ max_groups -1])
+will be the destination group for the message. If 
+.I max_groups
+is zero then even the destination group is not reported.
+
+The parameter
+.RI * mess_type
+will be set to the message type field the application sent with
+the original message, which is a restricted (see FL_multicast) 16 bit
+integer.  This value is endian corrected upon returning.
+
+The parameter
+.RI * endian_mismatch
+will be set to true (non-zero) if the endianness of the sending
+machine is the opposite of this receiving machine's.
+
+The actual message body being received will be filled into either the
+.I mess
+or
+.IR scat_mess 
+parameter.
+
+Flush Request Message:
+
+If this is a FLUSH_REQ_MESS then this represents that the underlying
+membership has changed and that this application needs to flush this
+group before the new view/membership can be installed (see
+FL_flush). 
+
+The
+.I sender
+will be filled in with the name of the group this connection needs
+to flush.  All the other fields are set to empty values.
+
+Membership Messages:
+
+If this is a MEMB_MESSAGE (i.e. membership message) and it
+specifically is a REG_MEMB_MESS, then:
+
+The 
+.I sender
+will be filled with the name of the group for which the membership
+change is occuring.
+
+The
+.I mess_type
+field will be set to the index of this process in the 
+.I groups
+array (see below). 
+
+The
+.I endian_mismatch
+field will be set to 0 since there are no endian issues with regular memberships.
+
+The
+.I groups
+array and message
+body are used to provide two kinds of membership information about the change that just
+occured in this group.  The 
+.I num_groups
+field will be set to the number of members in the group in the 
+.B new
+membership (i.e. after the change occured). Correspondingly, the 
+.I groups
+array will be filled in with the private group names of all members of this
+group in the
+.B new
+membership.  This list of names is always in the same order at all receipients
+and thus can be used to deterministically pick a group representative if
+one is needed by the application.
+
+The second set of information is stored in the message body.  The data
+buffer will include the following fields:
+.RS
+.TP
+.B group_id vid;
+.br
+.TP
+.B int num_members;
+.br
+.TP
+.B char dgroups[][MAX_GROUP_NAME];
+.RE
+
+The vid and num_members parameters will already be endian corrected.
+The dgroups array will contain num_members group names.  The content
+of the dgroups array is dependent upon the type of the membership
+change:
+.RS
+.TP 0.8i
+.B CAUSED_BY_JOIN:
+dgroups contains the private group of the joining process.
+.TP
+.B CAUSED_BY_LEAVE:
+dgroups contains the private group of the leaving process.
+.TP
+.B CAUSED_BY_DISCONNECT:
+dgroups contains the private group of the disconnecting process.
+.TP
+.B CAUSED_BY_NETWORK:
+dgroups contains the group names of the members of the new membership who came 
+with 
+.B this
+application to the new view/membership.  Using this data the previous
+membership listing and the new membership listing an application can
+determine all the members that left and were added in the NETWORK
+membership change.  Note, that a member can both leave and be added in
+a 
+.B single
+NETWORK change.
+.RE
+
+Transitional Message:
+
+If this is a MEMB_MESSAGE and specifically it is a TRANSITION_MESS,
+then this means that one or more of the current members of the group
+has left and not all of the safety guarantees can be met w.r.t. all of
+the original members for the messages delivered after this signal and
+before the next view/membership.  Transitional messages never have a
+CAUSED_BY type because they only serve to signal up to where SAFE
+delivery and AGREED delivery (with no holes) is guaranteed in the
+view/membership in which they are delivered.  Only one TRANSITION_MESS
+is delivered per group view/membership per connection.
+
+The 
+.I sender
+will be filled in with the name of the group for which the membership
+change is occurring.  All the other parameters will be set to empty values.
+
+Self-Leave Message:
+
+If this is a MEMB_MESSAGE (i.e. membership message) and it is neither
+a REG_MEMB_MESS or a TRANSITION_MESS, then it represents exactly the
+situtation where the member receiving this message has left a group
+and this is notification that the leave has occured, thus it is
+sometimes called a self-leave message.  The simplest test for this is
+if a message is CAUSED_BY_LEAVE and REG_MEMB_MESS is false (zero),
+then it is a self-leave message.
+
+The other members of the group that this member just left will receive
+a normal TRANSITION_MESS, REG_MEMB_MESS pair as described above
+showing this connection leaving.
+
+For self-leave messages the
+.I sender
+field will be filled in with the name of the group this connection is
+leaving.  All the other fields are set to empty values.
+
+.SH "RETURN VALUES"
+Returns the size of the data portion of the message received on
+success or one of the following errors ( < 0 ):
+.TP 0.8i
+.B ILLEGAL_SESSION
+The connection represented by 
+.I mbox
+was illegal, usually because it is not active.
+.TP
+.B ILLEGAL_PARAM
+An illegal parameter was passed, like a negative array size.
+.TP
+.B ILLEGAL_MESSAGE
+The msg buffer had an illegal structure, like an illegal number of
+scatter elements or a negatively sized buffer.
+.TP
+.B WOULD_BLOCK
+The receive call was made with the DONT_BLOCK service and it would
+have blocked because no messages were ready to be received.
+.TP
+.B GROUPS_TOO_SHORT
+The receive call was made with a groups array that was too small.  
+.RI * num_groups
+is filled in with the negative number of names that could have been
+returned.  Note, that the msg buffer could also be too short, and this
+should be determined by examining
+.I endian_mismatch
+as described above. If DROP_RECV was used this error code actually
+represents success (see above).
+.TP
+.B BUFFER_TOO_SHORT
+The receive call was made with a msg buffer that was too small.
+.RI * endian_mismatch
+is filled in with the negative number of bytes that could have been
+received if DROP_RECV isn't used.  If it is used then this error code
+actually represents success and 
+.I endian_mismatch
+has the normal meaning (see above).  Note, that the groups buffer
+could also be too short, this should be determined by examining
+.I num_groups
+as described above.
+.TP
+.B CONNECTION_CLOSED
+During communication to receive, communication errors occurred
+and the receive could not be completed.
+.SH AUTHOR
+John Schultz <jschultz at cnds.jhu.edu>

Added: trunk/docs/flush/man/FL_scat_multicast.3
===================================================================
--- trunk/docs/flush/man/FL_scat_multicast.3	2005-07-28 14:50:56 UTC (rev 252)
+++ trunk/docs/flush/man/FL_scat_multicast.3	2005-07-31 10:51:40 UTC (rev 253)
@@ -0,0 +1 @@
+.so FL_multicast.3

Added: trunk/docs/flush/man/FL_scat_receive.3
===================================================================
--- trunk/docs/flush/man/FL_scat_receive.3	2005-07-28 14:50:56 UTC (rev 252)
+++ trunk/docs/flush/man/FL_scat_receive.3	2005-07-31 10:51:40 UTC (rev 253)
@@ -0,0 +1 @@
+.so FL_receive

Added: trunk/docs/flush/man/FL_scat_subgroupcast.3
===================================================================
--- trunk/docs/flush/man/FL_scat_subgroupcast.3	2005-07-28 14:50:56 UTC (rev 252)
+++ trunk/docs/flush/man/FL_scat_subgroupcast.3	2005-07-31 10:51:40 UTC (rev 253)
@@ -0,0 +1 @@
+.so FL_multicast.3

Added: trunk/docs/flush/man/FL_scat_unicast.3
===================================================================
--- trunk/docs/flush/man/FL_scat_unicast.3	2005-07-28 14:50:56 UTC (rev 252)
+++ trunk/docs/flush/man/FL_scat_unicast.3	2005-07-31 10:51:40 UTC (rev 253)
@@ -0,0 +1 @@
+.so FL_multicast.3

Added: trunk/docs/flush/man/FL_subgroupcast.3
===================================================================
--- trunk/docs/flush/man/FL_subgroupcast.3	2005-07-28 14:50:56 UTC (rev 252)
+++ trunk/docs/flush/man/FL_subgroupcast.3	2005-07-31 10:51:40 UTC (rev 253)
@@ -0,0 +1 @@
+.so FL_multicast.3

Added: trunk/docs/flush/man/FL_unicast.3
===================================================================
--- trunk/docs/flush/man/FL_unicast.3	2005-07-28 14:50:56 UTC (rev 252)
+++ trunk/docs/flush/man/FL_unicast.3	2005-07-31 10:51:40 UTC (rev 253)
@@ -0,0 +1 @@
+.so FL_multicast.3

Added: trunk/docs/flush/man/FL_version.3
===================================================================
--- trunk/docs/flush/man/FL_version.3	2005-07-28 14:50:56 UTC (rev 252)
+++ trunk/docs/flush/man/FL_version.3	2005-07-31 10:51:40 UTC (rev 253)
@@ -0,0 +1,14 @@
+.TH FL_version 3 "Dec 2000" "Flush Spread" "User Manuals"
+
+.SH NAME
+FL_version \- returns library version information.
+.SH SYNOPSIS
+.B #include <fl.h>
+
+.BI "void FL_version(int *" major_ver ", int *" minor_ver ", int *" patch_ver ");"
+.SH DESCRIPTION
+.B FL_version
+returns the major, minor, and patch version numbers of the Flush
+Spread library in use.
+.SH AUTHOR
+John Schultz <jschultz at cnds.jhu.edu>

Added: trunk/flush/Makefile.in
===================================================================
--- trunk/flush/Makefile.in	2005-07-28 14:50:56 UTC (rev 252)
+++ trunk/flush/Makefile.in	2005-07-31 10:51:40 UTC (rev 253)
@@ -0,0 +1,57 @@
+CC=@CC@
+AR=@AR@
+
+SP_INC_DIR = -I../include
+SP_LIB_DIR = -L../daemon
+
+STD_INC_DIR = -I../stdutil/src
+STD_LIB_DIR = -L../stdutil/lib
+
+# use thread safe libs and enable std locking mechanisms (optional)
+THREAD_SAFE = -D_REENTRANT
+
+# don't use debugging (asserts and dprintfs) (optional)
+NO_DEBUG = -DNDEBUG
+
+# libraries apps need to link with: pthreads (optional), flush, stdutil, spread
+LIBS = @LIBS@ $(TARGET_DIR)/libflush.a -ltspread -lstdutil
+
+# where to put the libraries (libflush.a, libstdutil.a, etc.) and exes
+TARGET_DIR = .
+APP_DIR = .
+
+# all the defined variables
+DEFINES = $(THREAD_SAFE) $(NO_DEBUG) $(DO_TIMINGS)
+
+# all the include directories
+INCLUDES = $(SP_INC_DIR) -I. -I../stdutil/src
+
+# all the library linkage diretories
+LDFLAGS = $(SP_LIB_DIR)
+
+# gcc specific compiler flags
+CFLAGS = -Wall -O $(INCLUDES) $(DEFINES) # -g
+
+TARGETS = $(TARGET_DIR)/libflush.a $(APP_DIR)/flush_user $(APP_DIR)/sp_time_memb $(APP_DIR)/fl_time_memb
+
+OBJS = fl.o scatp.o
+
+all: $(TARGETS)
+
+$(TARGET_DIR)/libflush.a: $(OBJS)
+	$(AR) -rs $(TARGET_DIR)/libflush.a $(OBJS)
+
+$(APP_DIR)/flush_user: $(TARGET_DIR)/libflush.a user.o
+	$(CC) $(CFLAGS) $(LDFLAGS) -o $(APP_DIR)/flush_user user.o $(LIBS)
+
+$(APP_DIR)/sp_time_memb: $(TARGET_DIR)/libflush.a sp_time_memb.o stats.o
+	$(CC) $(CFLAGS) $(LDFLAGS) -o $(APP_DIR)/sp_time_memb sp_time_memb.o stats.o $(LIBS)
+
+$(APP_DIR)/fl_time_memb: $(TARGET_DIR)/libflush.a fl_time_memb.o stats.o
+	$(CC) $(CFLAGS) $(LDFLAGS) -o $(APP_DIR)/fl_time_memb fl_time_memb.o stats.o $(LIBS)
+
+clean:
+	rm -f *~ *.o $(TARGETS) core *.bak
+
+distclean: clean
+	rm -f Makefile config.*

Added: trunk/flush/configure.in
===================================================================
--- trunk/flush/configure.in	2005-07-28 14:50:56 UTC (rev 252)
+++ trunk/flush/configure.in	2005-07-31 10:51:40 UTC (rev 253)
@@ -0,0 +1,8 @@
+AC_INIT(fl.c)
+
+AC_PROG_CC
+AC_CHECK_PROGS(AR, ar)
+AC_CHECK_LIB(pthread, pthread_mutex_init)
+AC_CHECK_LIB(m, sqrt)
+
+AC_OUTPUT(Makefile)

Added: trunk/flush/fl.c
===================================================================
--- trunk/flush/fl.c	2005-07-28 14:50:56 UTC (rev 252)
+++ trunk/flush/fl.c	2005-07-31 10:51:40 UTC (rev 253)
@@ -0,0 +1,2141 @@
+/*
+ * The contents of this file are subject to the FLUSH SPREAD Non-Commercial 
+ * License, Version 1.0 (the ``License''); you may not use this file except 
+ * in compliance with the License. You may obtain a copy of the License at:
+ *
+ * http://www.cnds.jhu.edu/research/group/flush_spread/FLUSH_LICENSE
+ *
+ * or in the file ``FLUSH_LICENSE'' found in the root of this distribution.
+ *
+ * Software distributed under the License is distributed on an AS IS basis, 
+ * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License 
+ * for the specific language governing rights and limitations under the 
+ * License.
+ *
+ * The Original Software is:
+ *    The Flush Spread Library
+ *     
+ * The Initial Developers of the Original Software are:
+ *    Yair Amir, John Schultz and Jonathan Stanton
+ *
+ *    All Rights Reserved.
+ *
+ */
+
+#include <stdlib.h>
+
+#ifdef USE_DMALLOC
+#include <dmalloc.h>
+#endif
+
+#include <stdio.h>
+#include <stdarg.h>
+#include <string.h>
+#include <assert.h>
+#include <fl_p.h>
+#include <stdutil/stdutil.h>
+
+/* glob_conns is a look up table that maps mailboxes to fl_conn*s */
+static stdmutex glob_conns_lock;
+static stdhash  glob_conns      = STDHASH_STATIC_CONSTRUCT(sizeof(mailbox), sizeof(fl_conn*), NULL, NULL, 0);
+
+/********************************* public flush layer interface ********************************/
+
+int FL_lib_init(void)
+{
+  static stdbool first_time = STDTRUE;  /* *TRY* to mitigate bad call race conditions to FL_lib_init */
+  int            ret        = ILLEGAL_STATE;
+
+  if (first_time) {
+    first_time = STDFALSE;
+    ret = stdmutex_construct(&glob_conns_lock, STDMUTEX_FAST);
+  }
+
+  return ret;
+}
+
+void FL_version(int *major, int *minor, int *patch) { 
+  *major = FL_MAJOR_VERSION;
+  *minor = FL_MINOR_VERSION;
+  *patch = FL_PATCH_VERSION;
+}
+
+/* Establish a new fl connection. If SP_connect succceeds, create a new fl_conn instance. */
+int FL_connect(const char *daemon, const char *user, int priority, 
+			  mailbox *mbox, char private[MAX_GROUP_NAME]) {
+  int ret;
+  fl_conn *conn;        
+
+  DEBUG(std_stkfprintf(stderr, 1, "FL_connect: daemon '%s', user '%s', priority %d\n",
+		       daemon, user, priority));
+  if (FL_SP_version() < (float) 3.12) {             /* flush depends on the DROP_RECV semantics */
+    DEBUG(std_stkfprintf(stderr, 0, "REJECT_VERSION: SP too old v%f < v3.12\n", FL_SP_version()));
+    ret = REJECT_VERSION;
+  } else if ((ret = SP_connect(daemon, user, priority, 1, mbox, private)) == ACCEPT_SESSION) {
+    DEBUG(std_stkfprintf(stderr, 0, "mbox %d, private '%s'\n", *mbox, private));
+
+    if ((conn = (fl_conn*) calloc(1, sizeof(fl_conn))) == 0)
+      stderr_abort("(%s, %d): calloc(1, %u)\n", __FILE__, __LINE__, sizeof(fl_conn));
+    
+    stdmutex_construct(&conn->reserve_lock, STDMUTEX_FAST);
+    conn->reservations  = 0;
+    conn->disconnecting = 0;
+    stdcond_construct(&conn->destroy_cond);
+    
+    stdmutex_construct(&conn->recv_lock, STDMUTEX_FAST);
+    stdmutex_construct(&conn->conn_lock, STDMUTEX_FAST);
+    
+    conn->mbox       = *mbox;
+    conn->priority   = priority;
+    conn->group_memb = 1;
+    strncpy(conn->daemon, daemon, MAX_GROUP_NAME);
+    strncpy(conn->user, user, MAX_GROUP_NAME);
+    strncpy(conn->private, private, MAX_GROUP_NAME);
+    
+    stdhash_construct(&conn->groups, sizeof(char*), sizeof(fl_group*), 
+		      group_name_ptr_cmp, group_name_ptr_hashcode, 0);
+    stddll_construct(&conn->mess_queue, sizeof(gc_buff_mess*));             /* <gc_buff_mess*> */
+    conn->bytes_queued = 0;
+    
+    stdmutex_grab(&glob_conns_lock);                                         /* LOCK CONNS TAB */
+    stdhash_insert(&glob_conns, 0, mbox, &conn);                   /* add mbox -> conn mapping */
+    stdmutex_drop(&glob_conns_lock);                                     /* UNLOCK CONNS TAB */
+  }
+  DEBUG(std_stkfprintf(stderr, -1, "FL_connect: ret %d\n", ret));
+  return ret;
+}
+
+/* Destroy a mbox. This fcn recursively reclaims all resources
+   associated with a connection named mbox, in a synchronized, proper
+   fashion and returns whatever SP_disconnect does. If mbox doesn't
+   represent a valid fl connection, it just returns ILLEGAL_SESSION.  
+*/
+int FL_disconnect(mailbox mbox) {
+  stdit hit;
+  stdit lit;
+  fl_conn *conn;
+  int ret;
+
+  DEBUG(std_stkfprintf(stderr, 1, "FL_disconnect: mbox %d\n", mbox));
+  stdmutex_grab(&glob_conns_lock);                                           /* LOCK CONNS TAB */
+  if (!stdhash_is_end(&glob_conns, stdhash_find(&glob_conns, &hit, &mbox))) {              /* mbox found */
+    conn = *(fl_conn**) stdhash_it_val(&hit);
+    stdhash_erase(&glob_conns, &hit);                                 /* remove mbox -> fl_conn* mapping */
+    stdmutex_drop(&glob_conns_lock);                                     /* UNLOCK CONNS TAB */
+    /* now threads cannot come behind us and make_reservations() on this fl_conn */
+    
+    stdmutex_grab(&conn->reserve_lock);                                   /* LOCK RESERVE_LOCK */
+    conn->disconnecting = 1;                                 /* set disconnection indicator */
+    ret = SP_disconnect(mbox);                                /* unblock any blocked receivers */
+    
+    if (conn->reservations != 0) {                          /* WAIT if connection still in use */
+      DEBUG(std_stkfprintf(stderr, 0, "Waiting for mbox(%d, %p) to no longer be in use: "
+			   "%d reservations!\n", mbox, conn, conn->reservations));
+      stdcond_wait(&conn->destroy_cond, &conn->reserve_lock);
+    }
+    assert(conn->reservations == 0);                         /* should be no more reservations */
+
+    /* now, no other threads are using the connection -> safe to reclaim */
+    DEBUG(std_stkfprintf(stderr, 0, "Mbox(%d, %p) no longer in use: reclaiming!\n", mbox, conn));
+    stdmutex_drop(&conn->reserve_lock);                               /* UNLOCK RESERVE_LOCK */
+    
+    stdmutex_destruct(&conn->reserve_lock);  /* recursively destroy fl_conn and sub-structures */
+    stdcond_destruct(&conn->destroy_cond);
+    stdmutex_destruct(&conn->recv_lock);
+    stdmutex_destruct(&conn->conn_lock);
+    
+    for (stdhash_begin(&conn->groups, &hit); !stdhash_is_end(&conn->groups, &hit); stdhash_it_next(&hit))
+      free_fl_group(*(fl_group**) stdhash_it_val(&hit));
+    stdhash_destruct(&conn->groups);
+    
+    for (stddll_begin(&conn->mess_queue, &lit); !stddll_is_end(&conn->mess_queue, &lit); stddll_it_next(&lit))
+      free_gc_buff_mess(*(gc_buff_mess**) stddll_it_val(&lit));  
+    stddll_destruct(&conn->mess_queue);
+    
+    free(conn);
+  } else {
+    stdmutex_drop(&glob_conns_lock);                                     /* UNLOCK CONNS TAB */
+    DEBUG(std_stkfprintf(stderr, 0, "FL ILLEGAL_SESSION(%d)\n", mbox));
+    ret = ILLEGAL_SESSION; 
+  }
+  DEBUG(std_stkfprintf(stderr, -1, "FL_disconnect: ret %d\n", ret));
+  return ret;                                             /* return SP_disconnect return value */
+}
+
+int FL_join(mailbox mbox, const char *grp) {
+  fl_conn  *conn;
+  fl_group *group;
+  int ret;
+
+  DEBUG(std_stkfprintf(stderr, 1, "FL_join: mbox %d, group '%s'\n", mbox, grp));
+  if ((conn = lock_conn(mbox)) != 0) {
+    /* ACHTUNG! You may not join a group when you are already involved with that group */
+    if ((group = get_group(conn, grp)) == 0) {
+      ret = SP_join(mbox, grp);
+      unlock_conn(conn);
+      if (ret != 0) {
+	if (ret == CONNECTION_CLOSED || ret == ILLEGAL_SESSION)
+	  FL_disconnect(mbox);
+	else if (ret != ILLEGAL_GROUP)
+	  stderr_abort("(%s, %d): mbox %d: group %s: SP_join: unexpected error %d\n",
+		       __FILE__, __LINE__, mbox, grp, ret);
+      }
+    } else {
+      ret = ILLEGAL_GROUP;
+      unlock_conn(conn);
+      DEBUG(std_stkfprintf(stderr, 0, "FL ILLEGAL_GROUP('%s', %p) in mstate %d\n", grp, 
+			   group, group == 0 ? -1 : group->mstate));
+    }
+  } else {
+    DEBUG(std_stkfprintf(stderr, 0, "FL ILLEGAL_SESSION(%d)\n", mbox));
+    ret = ILLEGAL_SESSION;
+  }
+  DEBUG(std_stkfprintf(stderr, -1, "FL_join: ret %d\n", ret));
+  return ret;
+}
+
+int FL_leave(mailbox mbox, const char *grp) {
+  fl_conn  *conn;
+  fl_group *group;
+  int ret;
+
+  DEBUG(std_stkfprintf(stderr, 1, "FL_leave: mbox %d, group '%s'\n", mbox, grp));
+  if ((conn = lock_conn(mbox)) != 0) {
+    /* ACHTUNG! If you are a joined member of a fl group only then you can leave it only once! */
+    if ((group = get_group(conn, grp)) != 0 && group->mstate == JOINED) {
+      group->mstate = LEAVING;
+      ret = SP_leave(mbox, grp);
+      unlock_conn(conn);
+      if (ret != 0) {
+	if (ret == CONNECTION_CLOSED || ret == ILLEGAL_SESSION)
+	  FL_disconnect(mbox);
+	else if (ret == ILLEGAL_GROUP)
+	  stderr_abort("(%s, %d): mbox %d: group %s: SP_leave: ILLEGAL_GROUP\n", 
+		       __FILE__, __LINE__, mbox, grp);
+	else
+	  stderr_abort("(%s, %d): mbox %d: group %s: SP_leave: unexpected error %d\n", 
+		       __FILE__, __LINE__, mbox, grp, ret);
+      }
+    } else {
+      ret = ILLEGAL_GROUP;
+      unlock_conn(conn);
+      DEBUG(std_stkfprintf(stderr, 0, "FL ILLEGAL_GROUP('%s', %p) in mstate %d\n", grp, 
+			   group, group == 0 ? -1 : group->mstate));
+    }
+  } else {
+    DEBUG(std_stkfprintf(stderr, 0, "FL ILLEGAL_SESSION(%d)\n", mbox));
+    ret = ILLEGAL_SESSION;
+  }
+  DEBUG(std_stkfprintf(stderr, -1, "FL_leave: ret %d\n", ret));
+  return ret;
+}
+
+int FL_flush(mailbox mbox, const char *grp) {
+  fl_conn *conn;
+  fl_group *group;
+  int ret;
+
+  DEBUG(std_stkfprintf(stderr, 1, "FL_flush: mbox %d, group '%s'\n", mbox, grp));
+  if ((conn = lock_conn(mbox)) != 0) {
+    if ((group = get_group(conn, grp)) != 0)
+      ret = FL_int_flush(conn, group);
+    else {
+      DEBUG(std_stkfprintf(stderr, 0, "FL ILLEGAL_GROUP('%s', 0)\n", grp));
+      ret = ILLEGAL_GROUP; 
+    } 
+    unlock_conn(conn); 
+    if (ret == CONNECTION_CLOSED || ret == ILLEGAL_SESSION)
+      FL_disconnect(mbox);
+  } else {
+    DEBUG(std_stkfprintf(stderr, 0, "FL ILLEGAL_SESSION(%d)\n", mbox));
+    ret = ILLEGAL_SESSION;
+  }
+  DEBUG(std_stkfprintf(stderr, -1, "FL_flush: ret %d\n"));
+  return ret;
+}
+
+int FL_scat_subgroupcast(mailbox mbox, service serv_type, const char *grp,
+				int num_recvrs, char recvrs[][MAX_GROUP_NAME],
+				int16 mess_type, const scatter *scat) {
+  return FL_int_scat_multicast(mbox, serv_type | SUBGROUP_CAST, grp, 
+			       num_recvrs, recvrs, mess_type, scat);
+}
+
+int FL_subgroupcast(mailbox mbox, service serv_type, const char *grp,
+			   int num_recvrs, char recvrs[][MAX_GROUP_NAME],
+			   int16 mess_type, int mess_len, const char *mess) {
+  scatter msg;
+
+  msg.num_elements    = 1;
+  msg.elements[0].len = mess_len;
+  msg.elements[0].buf = (char*) mess;
+
+  return FL_scat_subgroupcast(mbox, serv_type, grp, num_recvrs, recvrs, mess_type, &msg);
+}
+
+int FL_scat_unicast(mailbox mbox, service serv_type, const char *grp, const char *recvr, 
+			   int16 mess_type, const scatter *scat) {
+  return FL_scat_subgroupcast(mbox, serv_type, grp, 1, (char(*)[MAX_GROUP_NAME]) recvr, 
+			      mess_type, scat);
+}
+
+int FL_unicast(mailbox mbox, service serv_type, const char *grp, const char *recvr, 
+		      int16 mess_type, int mess_len, const char *mess) {
+  scatter msg;
+
+  msg.num_elements    = 1;
+  msg.elements[0].len = mess_len;
+  msg.elements[0].buf = (char*) mess;
+
+  return FL_scat_unicast(mbox, serv_type, grp, recvr, mess_type, &msg);
+}
+
+int FL_scat_multicast(mailbox mbox, service serv_type, const char *grp,
+			     int16 mess_type, const scatter *scat) {
+  return FL_int_scat_multicast(mbox, serv_type & ~SUBGROUP_CAST, grp, 0, 0, mess_type, scat);
+}
+
+int FL_multicast(mailbox mbox, service serv_type, const char *grp,
+			int16 mess_type, int mess_len, const char *mess) {
+  scatter msg;
+
+  msg.num_elements    = 1;
+  msg.elements[0].len = mess_len;
+  msg.elements[0].buf = (char*) mess;
+
+  return FL_scat_multicast(mbox, serv_type, grp, mess_type, &msg);
+}
+
+int FL_scat_receive(mailbox mbox, service *serv_type, char *sender, int max_groups, 
+			   int *num_groups, char groups[][MAX_GROUP_NAME], int16 *mess_type,
+			   int *endian_mismatch, scatter *scat_mess, int *more_messes) {
+  int blocking = (*serv_type & DONT_BLOCK) == 0;              /* user not using DONT_BLOCK */ 
+  gc_recv_mess msg;                                    /* used to contain user's parameters */
+  fl_conn *conn;
+
+  DEBUG(std_stkfprintf(stderr, 1, "FL_scat_receive: mbox %d, service 0x%X, max_groups %d\n", 
+		       mbox, *serv_type, max_groups));
+
+  if (max_groups < 0) {
+    DEBUG(std_stkfprintf(stderr, 0, "Illegal max_groups!\n"));
+    msg.ret = ILLEGAL_PARAM;
+  } else {
+    /* I use msg rather than the individual parameters to pass user's variables around */
+    msg.mbox            = mbox;
+    msg.orig_serv_type  = *serv_type;
+    msg.serv_type       = serv_type;
+    msg.sender          = sender;
+    msg.max_groups      = max_groups;
+    msg.num_groups      = num_groups;
+    msg.groups          = groups;
+    msg.mess_type       = mess_type;
+    msg.endian_mismatch = endian_mismatch;
+    msg.scat_mess       = scat_mess;
+    msg.delivered       = 0;       /* does msg contain values to be returned to the user? */
+
+    if ((conn = make_reservation(mbox)) != 0) {      /* get conn and make sure it stays valid */
+      /* NEED TO ENFORCE THAT THERE IS ONLY ONE READING THREAD AT A TIME PER CONNECTION,      */
+      /* BECAUSE A THREAD COULD BECOME BLOCKED IN SP_recv AND THEN MSGS ARE GENERATED BY      */
+      /* ANOTHER THREAD AND PUT INTO THE QUEUE TO BE RETURNED -> THE BLOCKED THREAD           */
+      /* WOULDN'T REALIZE IT! SERIALIZATION ON RECV_LOCK ACCOMPLISHES THIS!!!!                */
+      if (acquire_recv_lock(conn)) {
+	if (acquire_conn_lock(conn)) {     /* get conn lock for so I can read connection info */
+	  int have_conn_lock = 1;                             /* do I have the conn lock? */
+	  
+	FL_scat_re_receive:
+	  msg.vulnerable           = 0;           /* is the contained message vulnerable? */
+	  msg.num_new_grps         = 0;              /* ensure new_grps and new_msg are empty */
+	  msg.new_msg.num_elements = 0;       /* used for DROP_RECV msgs (see FL_int_receive) */
+	  
+	  if (stddll_empty(&conn->mess_queue)) {                           /* empty msg queue */
+	    if (blocking || SP_poll(mbox) != 0) {       /* blocking allowed, or msg available */
+	      have_conn_lock = recv_and_handle(conn, &msg);     /* call SP_recv and enter FSA */
+	      
+	      if (msg.num_new_grps != 0)                     /* reclaim any allocated buffers */
+		free(msg.new_grps);
+	      
+	      if (msg.new_msg.num_elements != 0)
+		free(msg.new_msg.elements[0].buf);
+	      
+	      if (!msg.delivered) {               /* if msg doesn't contain returnable values */
+		DEBUG(std_stkfprintf(stderr, 0, "No return msg, going to top of recv loop!\n"));
+		goto FL_scat_re_receive;                   /* need to try and get a msg again */
+	      } else
+		DEBUG(std_stkfprintf(stderr, 0, "Msg contains values for user!\n"));
+	    } else { 
+	      DEBUG(std_stkfprintf(stderr, 0, "Call to receive would have blocked!\n"));
+	      msg.ret = WOULD_BLOCK;                                    /* would have blocked */
+	    }
+	  } else {                                                       /* message in queue! */
+	    gc_buff_mess *mess;                                              /* get buff mess */
+	    stdit lit;
+
+	    DEBUG(std_stkfprintf(stderr, 0, "Msg in buffer, reading in to user's parameters\n"));
+	    mess = *(gc_buff_mess**) stddll_it_val(stddll_begin(&conn->mess_queue, &lit));  
+	    if (buffm_to_userm(&msg, mess)) {     /* msg copy to user's parameters successful */
+	      stddll_erase(&conn->mess_queue, &lit);                     /* pop mess off of the top of the queue */
+	      conn->bytes_queued -= mess->total_size;                  /* update bytes queued */
+	      free_gc_buff_mess(mess);                            /* destroy the gc_buff_mess */
+	    } else
+	      DEBUG(std_stkfprintf(stderr, 0, "Msg incompatible with user's parameters!\n"));
+	  }
+	  if (have_conn_lock) {                             /* if I didn't lose the conn lock */
+	    if (msg.ret >= 0) 
+	      *more_messes = (int) stddll_size(&conn->mess_queue);      /* how many msgs left */
+	    release_conn_lock(conn);                               /* let go of the conn lock */
+	  }
+	} else { 
+	  DEBUG(std_stkfprintf(stderr, 0, "Couldn't acquire conn lock -> disconnecting!\n"));
+	  msg.ret = ILLEGAL_SESSION;
+	}
+	release_recv_lock(conn);
+      } else {
+	DEBUG(std_stkfprintf(stderr, 0, "Couldn't acquire recv lock -> disconnecting!\n"));
+	msg.ret = ILLEGAL_SESSION;
+      }
+      cancel_reservation(conn);
+    } else {
+      DEBUG(std_stkfprintf(stderr, 0, "Couldn't get a reservation\n"));
+      msg.ret = ILLEGAL_SESSION;
+    }
+    DEBUG(                                        /* only execute if statment when DEBUGGING */
+    if (msg.ret >= 0) {
+      int i;
+      
+      std_stkfprintf(stderr, 0, "Groups(%d):\n", *num_groups);
+      for (i = 0; i < *num_groups; ++i)
+	std_stkfprintf(stderr, 0, "\t'%s'\n", groups[i]);
+    }); 
+    if (msg.ret == CONNECTION_CLOSED || msg.ret == ILLEGAL_SESSION)
+      FL_disconnect(mbox);
+  }
+  DEBUG(std_stkfprintf(stderr, -1, "FL_scat_receive: ret %d, mbox %d, service 0x%X, sender '%s'"
+		       ", num_groups %d, mess_type %d, endian %d, more_messes %d\n", 
+		       msg.ret, mbox, *serv_type, msg.ret >= 0 ? sender : "", *num_groups, 
+		       *mess_type, *endian_mismatch, *more_messes));
+  return msg.ret;
+}
+
+int FL_receive(mailbox mbox, service *serv_type, char *sender, int max_groups, 
+		      int *num_groups, char groups[][MAX_GROUP_NAME], int16 *mess_type, 
+		      int *endian_mismatch, int max_mess_len, char *mess, int *more_messes) {
+  scatter msg;
+
+  msg.num_elements    = 1;
+  msg.elements[0].len = max_mess_len;
+  msg.elements[0].buf = (char*) mess;
+
+  return FL_scat_receive(mbox, serv_type, sender, max_groups, num_groups, groups, 
+			 mess_type, endian_mismatch, &msg, more_messes);
+}
+
+int FL_more_msgs(mailbox mbox) {
+  fl_conn *conn;
+  int ret;
+
+  DEBUG(std_stkfprintf(stderr, 1, "FL_more_msgs: mbox %d\n", mbox));
+  if ((conn = lock_conn(mbox)) != 0) {
+    ret = (int) stddll_size(&conn->mess_queue);
+    unlock_conn(conn);
+  } else {
+    DEBUG(std_stkfprintf(stderr, 0, "FL ILLEGAL_SESSION(%d)\n", mbox));
+    ret = ILLEGAL_SESSION;
+  }
+  DEBUG(std_stkfprintf(stderr, -1, "FL_more_msgs: ret %d\n", ret));
+  return ret;
+}
+
+int FL_poll(mailbox mbox) {
+  fl_conn *conn;
+  int ret;
+
+  DEBUG(std_stkfprintf(stderr, 1, "FL_poll: mbox %d\n", mbox));
+  if ((conn = lock_conn(mbox)) != 0) {
+    ret = conn->bytes_queued + SP_poll(mbox);
+    unlock_conn(conn);
+  } else {
+    DEBUG(std_stkfprintf(stderr, 0, "FL ILLEGAL_SESSSION(%d)\n", mbox));
+    ret = ILLEGAL_SESSION;
+  }
+  DEBUG(std_stkfprintf(stderr, -1, "FL_poll: ret %d\n", ret));
+  return ret;
+}
+
+void FL_error(int err) { 
+  switch (err) {
+  case ILLEGAL_PARAM:
+    printf("FL_error: (%d) Illegal parameter (eg a negative size) passed to a function.\n", err);
+    break;
+  case WOULD_BLOCK:
+    printf("FL_error: (%d) Function call would have blocked.\n", err);
+    break;
+  case ILLEGAL_MESSAGE_TYPE:
+    printf("FL_error: (%d) Illegal message type (int16) used, "
+	   "value < FL_MIN_LEGAL_MESS_TYPE(%d).\n", err, FL_MIN_LEGAL_MESS_TYPE);
+    break;
+  case ILLEGAL_STATE:
+    printf("FL_error: (%d) Function call peformed in a prohibited state.\n", err);
+    break;
+  case ILLEGAL_RECEIVERS:
+    printf("FL_error: (%d) Illegal receivers specified.\n", err);
+    break;
+  default:
+    SP_error(err);
+    break;
+  }
+}
+
+/************************************ private interface ****************************************/
+
+/* structure routines */
+/* When I initially create a group structure I set it up to be ready to insert in to a groups  */
+/* hash of a connection. I set mstate to JOINING indicating this group hasn't installed this   */
+/* connection as a member yet. I set vstate to AGREE, which will trigger auto-flush oks from   */
+/* this joining connection on SP memb events, until it finally gets installed. I create a      */
+/* fl_view with only the connection in fl_view->membs. I do this for three reasons: (1)        */
+/* sp_view and fl_view are always pointing at valid views now (no special checks for null),    */
+/* (2) in case this connection is joined through a NETWORK event, it will report the proper    */
+/* singleton vs set, and (3) this also gives me correct behavior on delivery of user's data    */
+/* msgs (is sender in my current fl view?).                                                    */
+static fl_group *create_fl_group(const char *conn_name, const char *group_name) {
+  fl_group *group;
+  group_id gid = {};
+
+  DEBUG(std_stkfprintf(stderr, 1, "create_fl_group: conn_name '%s', group '%s'\n", 
+		       conn_name, group_name));
+
+  if ((group = (fl_group*) calloc(1, sizeof(fl_group))) == 0)
+    stderr_abort("(%s, %d): calloc(1, %u)\n", __FILE__, __LINE__, sizeof(fl_group));
+
+  strncpy(group->group, group_name, MAX_GROUP_NAME);
+  group->mstate = JOINING;                                          /* not a JOINED member yet */
+  group->vstate = AGREE;        /* when I handle_next_memb_change first time -> sends FLUSH_OK */
+  group->curr_change = 0;
+  group->sp_view = group->fl_view = create_view(gid);            /* put an empty view to start */
+  fill_view(group->fl_view, 0, 1, (char(*)[MAX_GROUP_NAME]) conn_name, 0);      /* insert self */
+  group->fl_view->in_trans_memb = 1;         /* don't deliver any trans sigs until a member */
+  group->flush_recvs = 0;
+  stddll_construct(&group->mess_queue, sizeof(gc_buff_mess*));
+  stddll_construct(&group->memb_queue, sizeof(sp_memb_change*));
+  stdhash_construct(&group->pmemb_hash, sizeof(group_id), sizeof(sp_memb_change*),
+		    group_id_cmp, group_id_hashcode, 0); /* use default key fcns for group_id */
+  DEBUG(std_stkfprintf(stderr, -1, "create_fl_group: group('%s', %p)\n", group->group, group));
+  return group;
+}
+
+static void free_fl_group(fl_group *group) {
+  stdit lit;
+  stdit hit;
+
+  DEBUG(std_stkfprintf(stderr, 1, "free_fl_group: group('%s', %p)\n", group->group, group));
+  /* sp view is either equal to fl_view or points in to one of the sp_memb_change's */
+  free_view(group->fl_view);
+
+  for (stddll_begin(&group->mess_queue, &lit); !stddll_is_end(&group->mess_queue, &lit); stddll_it_next(&lit))
+    free_gc_buff_mess(*(gc_buff_mess**) stddll_it_val(&lit));
+  stddll_destruct(&group->mess_queue);
+
+  /* curr_change and all sp_memb_changes contained in memb_queue are also in pmemb */
+  stddll_destruct(&group->memb_queue);
+
+  for (stdhash_begin(&group->pmemb_hash, &hit); !stdhash_is_end(&group->pmemb_hash, &hit); stdhash_it_next(&hit))
+    free_sp_memb_change(*(sp_memb_change**) stdhash_it_val(&hit));
+  stdhash_destruct(&group->pmemb_hash);
+
+  free(group);
+  DEBUG(std_stkfprintf(stderr, -1, "free_fl_group\n"));
+}
+  
+static void free_gc_buff_mess(gc_buff_mess *msg) {
+  DEBUG(std_stkfprintf(stderr, 1, "free_gc_buff_mess: m %p, serv 0x%X\n", msg, msg->serv_type));
+  if (msg->num_groups)
+    free(msg->groups);
+  
+  if (msg->mess_len)
+    free(msg->mess);
+  
+  free(msg);
+  DEBUG(std_stkfprintf(stderr, -1, "free_gc_buff_mess\n"));
+}
+
+static sp_memb_change *create_sp_memb_change(group_id gid) {
+  sp_memb_change *spc;
+
+  DEBUG(std_stkfprintf(stderr, 1, "create_sp_memb_change: gid %d %d %d\n", 
+		       gid.id[0], gid.id[1], gid.id[2]));
+
+  if ((spc = (sp_memb_change*) calloc(1, sizeof(sp_memb_change))) == 0)
+    stderr_abort("(%s, %d): calloc(1, %u)\n", __FILE__, __LINE__, sizeof(sp_memb_change));
+
+  spc->memb_info       = create_view(gid);
+  spc->memb_mess_recvd = 0;
+  spc->memb_change_age = 0;
+  spc->delta_member    = 0;
+  stdhash_construct(&spc->flok_senders, MAX_GROUP_NAME, 0, 
+		    group_name_cmp, group_name_hashcode, 0);
+  spc->gid             = gid;
+
+  DEBUG(std_stkfprintf(stderr, -1, "create_sp_memb_change: ret %p, gid %d %d %d\n", spc,
+		       spc->memb_info->gid.id[0], spc->memb_info->gid.id[1], 
+		       spc->memb_info->gid.id[2])); 
+  return spc;
+}
+
+static void free_sp_memb_change(sp_memb_change *spc) {
+  DEBUG(std_stkfprintf(stderr, 1, "free_sp_memb_change: spc %p\n", spc));
+
+  if (spc->memb_info != 0)
+    free_view(spc->memb_info);
+
+  if (spc->delta_member != 0)
+    free(spc->delta_member);
+
+  stdhash_destruct(&spc->flok_senders);
+  free(spc);
+  DEBUG(std_stkfprintf(stderr, -1, "free_sp_memb_change\n"));
+}
+
+static view *create_view(group_id gid) {
+  view *v;
+
+  DEBUG(std_stkfprintf(stderr, 1, "create_view: gid %d %d %d\n", 
+		       gid.id[0], gid.id[1], gid.id[2]));
+
+  if ((v = (view*) calloc(1, sizeof(view))) == 0)
+    stderr_abort("(%s, %d): calloc(1, %u)\n", sizeof(view));
+
+  v->gid            = gid;
+  v->memb_type      = 0;
+  v->in_trans_memb  = 0;
+  v->orig_num_membs = 0;
+  v->membs_names    = 0;
+  stdhash_construct(&v->orig_membs, sizeof(char*), 0, 
+		    group_name_ptr_cmp, group_name_ptr_hashcode, 0);
+  stdhash_construct(&v->curr_membs, sizeof(char*), 0, 
+		    group_name_ptr_cmp, group_name_ptr_hashcode, 0);
+  v->my_index       = -1;
+
+  DEBUG(std_stkfprintf(stderr, -1, "create_view: ret %p\n", v));
+  return v;
+}
+
+static void free_view(view *v) {
+  DEBUG(std_stkfprintf(stderr, 1, "free_view: view %p, gid %d %d %d\n", v, v->gid.id[0],
+		       v->gid.id[1], v->gid.id[2]));
+  if (v->orig_num_membs != 0)
+    free(v->membs_names);
+  stdhash_destruct(&v->orig_membs);
+  stdhash_destruct(&v->curr_membs);
+  free(v);
+  DEBUG(std_stkfprintf(stderr, -1, "free_view\n"));
+}
+
+static void fill_view(view *v, service memb_type, int num_membs, 
+				 char (*membs)[MAX_GROUP_NAME], int16 index) {
+  size_t byte_size = num_membs * MAX_GROUP_NAME;
+
+  DEBUG(std_stkfprintf(stderr, 1, "fill_view: view %p, serv 0x%X, num_membs %d, index %d\n",
+		       v, memb_type, num_membs, index));
+
+  assert(num_membs > 0 && index >= 0 && index < num_membs && v->orig_num_membs == 0 &&
+	 stdhash_empty(&v->orig_membs) && stdhash_empty(&v->orig_membs));
+
+  v->memb_type = memb_type;
+  v->orig_num_membs = num_membs;
+
+  if ((v->membs_names = (char(*)[MAX_GROUP_NAME]) malloc(byte_size)) == 0)
+    stderr_abort("(%s, %d): malloc (%d)\n", __FILE__, __LINE__, byte_size);
+  memcpy(v->membs_names, membs, byte_size);
+
+  for (membs = v->membs_names; num_membs--; ++membs)        /* I set membs = v->membs_names!!! */
+    stdhash_insert(&v->orig_membs, 0, &membs, 0), stdhash_insert(&v->curr_membs, 0, &membs, 0);
+
+  v->my_index = index;
+  DEBUG(std_stkfprintf(stderr, -1, "fill_view: view %p, serv 0x%X, num_membs %lu, index %d\n",
+		       v, v->memb_type, stdhash_size(&v->orig_membs), v->my_index));
+}
+
+/* Looks up a fl_conn "named" mbox, gets an outstanding reservation on
+   it and acquires its conn_lock. Returns the valid locked fl_conn* on
+   success or 0 on failure -> ILLEGAL_SESSION.
+*/
+static fl_conn *lock_conn(mailbox mbox) {
+  fl_conn *conn;
+
+  if ((conn = make_reservation(mbox)) == 0)
+    return 0;
+
+  if (!acquire_conn_lock(conn)) /* error -> disconnecting */
+    return cancel_reservation(conn), (fl_conn*) 0;
+
+  return conn;
+}
+
+/* Unlock a connection successfully locked with lock_conn. */
+static void unlock_conn(fl_conn *conn) {
+  release_conn_lock(conn);
+  cancel_reservation(conn);
+}
+
+/* Get a pointer to a fl_conn "named" mbox and "acquire an outstanding
+   reservation" by increasing a reservation counter. A successful
+   return *ONLY* entitles the caller to make subsequent calls to
+   acquire/release_conn_lock and acquire/release_recv_lock on the
+   returned fl_conn. Each successful call to make_reservation MUST be
+   matched by a call to cancel_reservation on the returned
+   fl_conn. This outstanding reservation only ensures that the
+   returned fl_conn is valid between a successful call to
+   make_reservation and the subsequent call to cancel_reservation on
+   the returned fl_conn. If mbox is not a valid fl connection this fcn
+   fails and returns 0. NOTE: it is imperative that this fcn acquires
+   the reserve lock before releasing the glob_conns_lock.  
+*/
+static fl_conn *make_reservation(mailbox mbox) { 
+  stdit hit;
+  fl_conn *conn;
+
+  stdmutex_grab(&glob_conns_lock);                                 /* LOCK CONNS TAB */
+
+  if (stdhash_is_end(&glob_conns, stdhash_find(&glob_conns, &hit, &mbox)))     /* no such mbox */
+    return stdmutex_drop(&glob_conns_lock), (fl_conn*) 0;      /* UNLOCK CONNS TAB */
+
+  conn = *(fl_conn**) stdhash_it_val(&hit);
+  stdmutex_grab(&conn->reserve_lock);                           /* LOCK RESERVE_LOCK */
+  stdmutex_drop(&glob_conns_lock);                             /* UNLOCK CONNS TAB */
+  if (!conn->disconnecting)                                /* check if disconnecting */
+    ++conn->reservations;                               /* increment reservation cnt */
+  stdmutex_drop(&conn->reserve_lock);                       /* UNLOCK RESERVE_LOCK */
+
+  return !conn->disconnecting ? conn : 0;
+}
+
+/* Relinquish a reservation this thread owns on a connection. */
+static void cancel_reservation(fl_conn *conn) {
+  stdmutex_grab(&conn->reserve_lock);                           /* LOCK RESERVE_LOCK */
+  assert(conn->reservations != 0);
+  if (--conn->reservations == 0 && conn->disconnecting) /* decrement reservation cnt */
+    stdcond_wake_all(&conn->destroy_cond);               /* signal disconnector thread */
+  stdmutex_drop(&conn->reserve_lock);                       /* UNLOCK RESERVE_LOCK */
+}
+
+/* Acquire a connection's conn lock. The caller must have an
+   outstanding reservation on conn. Once this lock is acquired, a
+   thread can freely examine and/or modify the data in conn as
+   indicated in <fl_p.h>. Returns 0 on failure -> ILLEGAL_SESSION.
+   Each successful call made by a thread on a particular conn must be
+   matched with a call to release_conn_lock to allow that conn to be
+   used by other threads. A thread may not make a second call to
+   acquire_conn_lock after a successful return on a particular conn
+   without an intervening call to release_conn_lock on that conn to
+   prevent self-deadlocks.  
+*/
+static int acquire_conn_lock(fl_conn *conn) {
+  stdmutex_grab(&conn->conn_lock);                                 /* LOCK CONN_LOCK */
+  if (conn->disconnecting)                                  /* conn is disconnecting */
+    return stdmutex_drop(&conn->conn_lock), 0;
+
+  return 1;
+}
+
+/* Release a connection's conn lock. The caller must have an
+   oustanding reservation on release. The calling thread must also
+   have made a successful call to acquire_conn_lock on release without
+   any intervening calls to release_conn_lock on release.  
+*/
+static void release_conn_lock(fl_conn *release) {
+  stdmutex_drop(&release->conn_lock);                            /* UNLOCK CONN_LOCK */
+}
+
+/* Acquire a connection's recv lock. The caller must have an
+   outstanding reservation on conn. Once this lock is acquired, a
+   thread can proceed into the body of FL_scat_recv. Returns 0
+   on failure -> ILLEGAL_SESSION. Each successful call made by a
+   thread on a particular conn must be matched with a call to
+   release_recv_lock to allow that conn to be used by other threads. A
+   thread may not make a second call to acquire_recv_lock after a
+   successful return on a particular conn without an intervening call
+   to release_recv_lock on that conn to prevent self-deadlocks.  NOTE
+   THAT THIS LOCK DOES NOT GIVE A THREAD ANY RIGHTS TO EXAMINE OR
+   MODIFY _ANY_ OF THE CONNECTION'S DATA!! IT ONLY ALLOWS A THREAD TO
+   ENTER THE BODY OF FL_scat_recv!!  
+*/
+static int acquire_recv_lock(fl_conn *conn) {
+  stdmutex_grab(&conn->recv_lock);           /* LOCK RECV_LOCK */
+  if (conn->disconnecting)                   /* conn is disconnecting */
+    return stdmutex_drop(&conn->recv_lock), 0;
+
+  return 1;
+}
+
+/* Release a connection's recv lock. The caller must have an
+   oustanding reservation on release. The calling thread must have
+   made a successful call to acquire_recv_lock on release without any
+   intervening calls to release_recv_lock on release.  
+*/
+static void release_recv_lock(fl_conn *release) {
+  stdmutex_drop(&release->recv_lock);           /* UNLOCK RECV_LOCK */
+}
+
+static fl_group *add_group(fl_conn *conn, const char *group_name) {
+  fl_group *group;
+  char *group_name_ptr;
+
+  DEBUG(std_stkfprintf(stderr, 1, "add_group: mbox(%d, %p), group '%s'\n", 
+		       conn->mbox, conn, group_name));
+  group = create_fl_group(conn->private, group_name);
+  group_name_ptr = group->group;
+  stdhash_insert(&conn->groups, 0, &group_name_ptr, &group);
+  DEBUG(std_stkfprintf(stderr, -1, "add_group: return group('%s', %p)\n", group->group, group));
+  return group;
+}
+
+static void remove_group(fl_conn *conn, fl_group *group) {
+  stdit hit;
+  char *group_name_ptr;
+
+  DEBUG(std_stkfprintf(stderr, 1, "remove_group: mbox(%d, %p), group ('%s', %p)\n",
+		       conn->mbox, conn, group->group, group));
+  group_name_ptr = group->group;
+  stdhash_find(&conn->groups, &hit, &group_name_ptr);
+  assert(!stdhash_is_end(&conn->groups, &hit) && group == *(fl_group**) stdhash_it_val(&hit));
+  stdhash_erase(&conn->groups, &hit);
+  free_fl_group(group);
+  DEBUG(std_stkfprintf(stderr, -1, "remove_group\n"));
+}
+
+static fl_group *get_group(fl_conn *conn, const char *grp) {
+  stdit hit;
+
+  return (!stdhash_is_end(&conn->groups, stdhash_find(&conn->groups, &hit, &grp)) ? 
+	  *(fl_group**) stdhash_it_val(&hit) : 0);
+}
+
+/* Pass null for bm if the data to be delivered is already in um.                         */
+/* Otherwise fill out a gc_buff_mess with the appropriate data and indicate whether       */
+/* that buff mess is dynamically allocated or not. If it is, then if the message needs to */
+/* be buffered, then bm itself is buffered. If the dynamically allocated message doesn't  */
+/* need to be buffered, then the msg is reclaimed (freed) before returning to the caller. */
+/* Returns a pointer to the gc_buff_mess that was buffered, null if none was buffered.    */
+/* um can be null if you force buffering somehow.                                         */
+static gc_buff_mess *deliver(fl_conn *conn, gc_recv_mess *um, gc_buff_mess *bm,
+					int bm_alloced) {
+  gc_buff_mess *ret;
+
+  DEBUG(std_stkfprintf(stderr, 1, "deliver: mbox(%d, %p), um %p, bm %p, bm_alloced %d\n",
+		       conn->mbox, conn, um, bm, (int) bm_alloced));
+  if (!um->delivered && stddll_empty(&conn->mess_queue) && (bm == 0 || buffm_to_userm(um, bm))) {
+    DEBUG(std_stkfprintf(stderr, 0, "able to deliver to user's parameters!\n"));
+    um->delivered = 1;
+    ret = 0;
+    if (bm != 0 && bm_alloced) {
+      DEBUG(std_stkfprintf(stderr, 0, "bm was alloc'ed -> freeing %p\n", bm));
+      free_gc_buff_mess(bm);
+    }
+  } else {
+    DEBUG(std_stkfprintf(stderr, 0, "UNABLE to deliver to user's parameters: BUFFERING!\n"));
+    if (bm == 0 || !bm_alloced) {                    /* if I need to alloc a gc_buff_mess */
+      if ((ret = (gc_buff_mess*) malloc(sizeof(gc_buff_mess))) == 0)
+	stderr_abort("(%s, %d): malloc(%u)\n", __FILE__, __LINE__, sizeof(gc_buff_mess));
+
+      if (bm == 0) {                          /* if data was contained in user parameters */
+	DEBUG(std_stkfprintf(stderr, 0, "data was in user's parameters copying to buff mess\n"));
+	userm_to_buffm(ret, um);
+      } else {                                           /* else data was contained in bm */
+	DEBUG(std_stkfprintf(stderr, 0, "data was in bm, copying in to buff mess\n"));
+	*ret = *bm;
+      }
+    } else {                             /* buff mess was already alloc'ed and filled out */
+      DEBUG(std_stkfprintf(stderr, 0, "data in bm, but alloc'ed: stealing bm for buffer\n"));
+      ret = bm;
+    }
+    /* calculate total size of buffered message */
+    ret->total_size = sizeof(gc_buff_mess) + ret->num_groups * MAX_GROUP_NAME + ret->mess_len;
+    stddll_push_back(&conn->mess_queue, &ret);
+    conn->bytes_queued += ret->total_size;
+  }
+  DEBUG(std_stkfprintf(stderr, -1, "deliver: ret buff mess %p\n", ret));
+  return ret;
+}
+
+static gc_buff_mess *deliver_trans_sig(fl_conn *conn, gc_recv_mess *um, char *group) {
+  gc_buff_mess trans_sig = {}, *ret; /* sets everything to zero */
+
+  DEBUG(std_stkfprintf(stderr, 1, "deliver_trans_sig: mbox(%d, %p), group '%s'\n",
+		       conn->mbox, conn, group));
+  trans_sig.mbox = conn->mbox;
+  trans_sig.serv_type = TRANSITION_MESS;
+  strncpy(trans_sig.sender, group, MAX_GROUP_NAME);
+  ret = deliver(conn, um, &trans_sig, 0);
+
+  DEBUG(std_stkfprintf(stderr, -1, "deliver_trans_sig: ret buff mess %p\n", ret));
+  return ret;
+}
+
+static gc_buff_mess *deliver_flush_req(fl_conn *conn, gc_recv_mess *um, char *group) {
+  gc_buff_mess flush_req = {}, *ret; /* sets everything to zero */
+
+  DEBUG(std_stkfprintf(stderr, 1, "deliver_flush_req: mbox(%d, %p), group '%s'\n",
+		       conn->mbox, conn, group));
+  flush_req.mbox = conn->mbox;
+  flush_req.serv_type = FLUSH_REQ_MESS;
+  strncpy(flush_req.sender, group, MAX_GROUP_NAME);
+  ret = deliver(conn, um, &flush_req, 0);
+
+  DEBUG(std_stkfprintf(stderr, -1, "deliver_flush_req: ret buff mess %p\n", ret));
+  return ret;
+}
+
+/* copy info from buffered msg to a user's parameters and make it presentable to the user  */
+/* return 1 on successful copy to user, 0 on failure (incompatible user parameters) */
+static int buffm_to_userm(gc_recv_mess *um, const gc_buff_mess *bm) {
+  int ret = 1;
+  scatp pos;
+  long err;
+
+  DEBUG(std_stkfprintf(stderr, 1, "buffm_to_userm: um %p, bm %p, serv 0x%X, sender '%s'\n", 
+		       um, bm, bm->serv_type, bm->sender));
+
+  if (scatp_begin(&pos, um->scat_mess) == 0 &&
+      (err = scatp_cpy2(&pos, bm->mess, bm->mess_len)) >= 0) {      /* user's scatter is legal */
+    um->mbox       = bm->mbox;
+    *um->serv_type = bm->serv_type;
+    *um->mess_type = bm->mess_type;
+    
+    if (err == bm->mess_len && bm->num_groups <= um->max_groups) {        /* no buffer problem */
+      *um->num_groups = bm->num_groups;
+      strncpy(um->sender, bm->sender, MAX_GROUP_NAME);
+      memcpy(um->groups, bm->groups, 
+	     (*um->num_groups >= 0 ? *um->num_groups : um->max_groups) * MAX_GROUP_NAME);
+      *um->endian_mismatch = bm->endian_mismatch;
+      um->ret = bm->mess_len;
+    } else {                                                                 /* buffer problem */
+      DEBUG(std_stkfprintf(stderr, 0, "User buffer problem!\n"));
+      if ((um->orig_serv_type & DROP_RECV) == 0) {            /* user didn't request DROP_RECV */
+	DEBUG(std_stkfprintf(stderr, 0, "User didn't request DROP_RECV\n"));
+	if (err != bm->mess_len) {
+	  um->ret = BUFFER_TOO_SHORT;
+	  *um->endian_mismatch = -bm->mess_len;
+	} else
+	  *um->endian_mismatch = 0;
+	
+	if (bm->num_groups > um->max_groups) {
+	  um->ret = GROUPS_TOO_SHORT;
+	  *um->num_groups = -bm->num_groups;
+	} else
+	  *um->num_groups = 0;
+
+	ret = 0;                      /* error: buffer problem and DROP_RECV not requested */
+      } else {                                                     /* user requested DROP_RECV */
+	DEBUG(std_stkfprintf(stderr, 0, "User DID request DROP_RECV!\n"));
+	if (err != bm->mess_len)
+	  um->ret = BUFFER_TOO_SHORT;
+	
+	if (bm->num_groups > um->max_groups) {
+	  um->ret = GROUPS_TOO_SHORT;
+	  *um->num_groups = -bm->num_groups;
+	} else
+	  *um->num_groups = bm->num_groups;
+
+	strncpy(um->sender, bm->sender, MAX_GROUP_NAME);
+	memcpy(um->groups, bm->groups, 
+	       (*um->num_groups >= 0 ? *um->num_groups : um->max_groups) * MAX_GROUP_NAME);
+	*um->endian_mismatch = bm->endian_mismatch;
+      }
+    }
+  } else {                                                /* error: user's scat msg is illegal */
+    um->ret = ILLEGAL_MESSAGE;
+    ret = 0;
+  }
+  DEBUG(std_stkfprintf(stderr, -1, "buffm_to_userm: um %p, bm %p, serv 0x%X, sender '%s'\n", 
+		       um, bm, bm->serv_type, bm->sender));
+  return ret;
+}
+
+/* copy info from a user's parameters to a buffered message */
+static void userm_to_buffm(gc_buff_mess *bm, const gc_recv_mess *um) {
+  bm->mbox = um->mbox;
+  bm->serv_type = *um->serv_type;
+  strncpy(bm->sender, um->sender, MAX_GROUP_NAME);
+  bm->mess_type = *um->mess_type;
+  bm->endian_mismatch = *um->endian_mismatch;
+
+  DEBUG(std_stkfprintf(stderr, 1, "userm_to_buffm: ret %d, vuln %d, delivered %d, serv 0x%X, "
+		       "sender '%s'\n", um->ret, (int) um->vulnerable, (int) um->delivered, 
+		       *um->serv_type, um->sender));
+  /* if um isn't an error without a msg body and groups */
+  if (um->ret >= 0 || um->ret == GROUPS_TOO_SHORT || um->ret == BUFFER_TOO_SHORT) { 
+    scatter *scat;
+    size_t groups_size;
+    scatp um_pos;
+    char *groups;
+    long err;
+
+    /* get groups info out of the msg: either in um->groups or um->new_grps */
+    get_groups_info(um, &bm->num_groups, (char(**)[MAX_GROUP_NAME]) &groups);
+    if ((groups_size = bm->num_groups * MAX_GROUP_NAME) != 0) {
+      if ((bm->groups = (char(*)[MAX_GROUP_NAME]) malloc(groups_size)) == 0)
+	stderr_abort("(%s, %d): malloc(%u)\n", __FILE__, __LINE__, groups_size);    
+      memcpy(bm->groups, groups, groups_size);
+    } else
+      bm->groups = 0;
+
+    /* get scat info out of the msg: either in um->scat_mess or um->new_msg */
+    get_scat_info(um, &bm->mess_len, &scat);
+    if (bm->mess_len != 0) {
+      if ((bm->mess = (char*) malloc(bm->mess_len)) == 0)
+	stderr_abort("(%s, %d): malloc(%d)\n", __FILE__, __LINE__, bm->mess_len);
+
+      err = scatp_begin(&um_pos, scat);
+      assert(err == 0);
+      err = scatp_cpy1(bm->mess, &um_pos, bm->mess_len);
+      assert(err == bm->mess_len);    
+    } else
+      bm->mess = 0;
+  } else {
+    stderr_abort("not sure about this path: if ever triggered think about it\n");
+    DEBUG(std_stkfprintf(stderr, 0, "copying an error message to a buffm: ret %d\n", um->ret));
+    bm->num_groups = 0;
+    bm->groups     = 0;
+    bm->mess_len   = um->ret;
+    bm->mess       = 0;
+  }
+  DEBUG(std_stkfprintf(stderr, -1, "userm_to_buffm\n"));
+}
+
+static int FL_int_flush(fl_conn *conn, fl_group *group) {
+  int ret;
+
+  DEBUG(std_stkfprintf(stderr, 1, "FL_int_flush: mbox(%d, %p), group ('%s', %p)\n",
+		       conn->mbox, conn, group->group, group));
+  /* ACHTUNG! You must be a non-leaving, AUTHORIZE member of a fl group to flush it */
+  if (group->mstate != LEAVING) {
+    if (group->vstate == AUTHORIZE) {
+      DEBUG(std_stkfprintf(stderr, 0, "Going to state AGREE: Actually sending "
+			   "the FLUSH_OK_MESS!\n"));
+      group->vstate = AGREE;
+      ret = SP_multicast(conn->mbox, FIFO_MESS, group->group, FLUSH_OK_MESS, 
+			 sizeof(group_id), (char*) &group->curr_change->memb_info->gid);
+
+      if (ret != sizeof(group_id)) {
+	if (ret != CONNECTION_CLOSED && ret != ILLEGAL_SESSION)
+	  stderr_abort("(%s, %d): mbox %d: group %s: SP_multicast: unexpected error(%d)\n", 
+		       __FILE__, __LINE__, conn->mbox, group->group, ret);
+      } else 
+	ret = 0;                                                     /* set to zero -> success */
+    } else {
+      DEBUG(std_stkfprintf(stderr, 0, "Group not in AUTHORIZE state!\n"));
+      ret = ILLEGAL_STATE;
+    }
+  } else {
+    DEBUG(std_stkfprintf(stderr, 0, "Group is in LEAVING state!\n"));
+    ret = ILLEGAL_GROUP;
+  }
+  DEBUG(std_stkfprintf(stderr, -1, "FL_int_flush: ret %d\n", ret));
+  return ret;
+}
+
+static int FL_int_scat_multicast(mailbox mbox, service serv_type, const char *grp,
+					int num_recvrs, char recvrs[][MAX_GROUP_NAME],
+					int16 mess_type, const scatter *user_scat) {
+  fl_conn *conn;
+  fl_group *group = 0;
+  int grp_not_priv;                                           /* is grp not a private group ? */
+  int ret, i;
+
+  int fix_scat = 0;                                      /* did I modify the user's scat? */
+  scatter *scat = (scatter*) user_scat;                           /* get rid of const warnings */
+  scat_element senders_elem;                   /* for copy of user's element I might overwrite */
+  char copy_buf[sizeof(group_id) + sizeof(int16) + MAX_GROUP_NAME];           /* append buffer */
+  char *curr_ptr = copy_buf, *key;                             /* current pos in append buffer */
+
+  DEBUG( /* debug print out user's parameters */
+	std_stkfprintf(stderr, 1, "FL_int_scat_multicast: mbox %d, serv 0x%X, grp '%s', "
+		       "mess_type %d, mess_len %ld\n", mbox, serv_type, grp, 
+		       mess_type, scat_capacity(scat));
+	std_stkfprintf(stderr, 0, "Recvrs(%d):\n", num_recvrs);
+	for (i = 0; i < num_recvrs; ++i) {
+	  std_stkfprintf(stderr, 0, "\t'%s'\n", recvrs[i]);
+	});
+
+  if ((conn = lock_conn(mbox)) != 0) {                                      /* LOCK CONNECTION */
+    if (num_recvrs < 0) {                                                       /* num_recvrs? */
+      DEBUG(std_stkfprintf(stderr, 0, "Illegal num_recvrs %d!\n", num_recvrs));
+      ret = ILLEGAL_PARAM;
+    } else if (IS_ILLEGAL_SEND_STYPE(serv_type)) {          /* used reserved bits in serv_type */
+      DEBUG(std_stkfprintf(stderr, 0, "Illegal use of reserved service bits\n"));
+      ret = ILLEGAL_SERVICE;
+    } else if (IS_ILLEGAL_SEND_MTYPE(mess_type)) {                /* used a reserved mess type */
+      DEBUG(std_stkfprintf(stderr, 0, "Illegal use of reserved message type!\n"));
+      ret = ILLEGAL_MESSAGE_TYPE;
+    } else if (scat->num_elements < 0 || scat->num_elements > FL_MAX_SCATTER_ELEMENTS) {
+      DEBUG(std_stkfprintf(stderr, 0, "Illegal scatter num_elements %d\n", scat->num_elements));
+      ret = ILLEGAL_MESSAGE;
+    } else if ((grp_not_priv = !is_private_group(grp)) &&   /* I allow sends to private groups */
+	       ((group = get_group(conn, grp)) == 0 || group->mstate != JOINED)) {
+      DEBUG(std_stkfprintf(stderr, 0, "Illegal send to group of which not a member!\n"));
+      ret = ILLEGAL_GROUP;                                       /* not a member of that group */
+    } else if (grp_not_priv && group->vstate == AGREE) {
+      DEBUG(std_stkfprintf(stderr, 0, "Illegal send: state AGREE: group '%s'!\n", group->group));
+      ret = ILLEGAL_STATE;
+    } else {          /* attempt to send msg: still need to check on recvrs for subgroup_casts */
+      if (grp_not_priv && is_vulnerable_mess(group, serv_type)) { /* mess type, current fl vid */
+	DEBUG(std_stkfprintf(stderr, 0, "Sending a vulnerable message!\n"));
+	fix_scat = 1;
+	memcpy(curr_ptr, &mess_type, sizeof(int16));
+	curr_ptr += sizeof(int16);
+	memcpy(curr_ptr, &group->fl_view->gid, sizeof(group_id));
+	curr_ptr += sizeof(group_id);
+	mess_type = VULNERABLE_MESS;
+      }
+      if (!Is_subgroup_mess(serv_type)) {                                  /* normal multicast */
+	if (fix_scat) {                   /* actually change the scatter to reflect appendings */
+	  senders_elem = scat->elements[scat->num_elements];            /* save user's element */
+	  scat->elements[scat->num_elements].buf = copy_buf;
+	  scat->elements[scat->num_elements].len = (int) (curr_ptr - copy_buf);
+	  ++scat->num_elements;
+	}
+	ret = SP_scat_multicast(mbox, serv_type, grp, mess_type, scat);
+      } else {                                                           /* subgroup multicast */
+	DEBUG(std_stkfprintf(stderr, 0, "Sending a subgroupcast!\n"));
+	if (grp_not_priv) {             /* check all recvrs were members of my current fl view */
+	  stdhash *orig_membs = &group->fl_view->orig_membs;
+	  stdit find;
+      
+	  for (i = 0; i < num_recvrs; ++i) { 
+	    key = (char*) (recvrs + i);
+	    if (stdhash_is_end(orig_membs, stdhash_find(orig_membs, &find, &key))) {  /* a recvr wasn't */
+	      DEBUG(std_stkfprintf(stderr, 0, "Illegal recvr '%s' not in ref '%s'\n", key, grp));
+	      break;
+	    }
+	  }
+	} else {                                        /* allow subgroupcast to private group */
+	  for (i = 0; i < num_recvrs; ++i) { 	  /* require ALL "receivers" to be same as grp */
+	    key = (char*) (recvrs + i);
+	    if (strncmp(grp, key, MAX_GROUP_NAME) != 0) {
+	      DEBUG(std_stkfprintf(stderr, 0, "Illegal recvr '%s' not in ref '%s'\n", key, grp));
+	      break;
+	    }
+	  }
+	}
+	fix_scat = 1;                    /* subgroup msgs have reference group appended to msg */
+	strncpy(curr_ptr, grp, MAX_GROUP_NAME);
+	curr_ptr += MAX_GROUP_NAME;
+
+	/* actually change the scatter to reflect appendings */
+	senders_elem = scat->elements[scat->num_elements];              /* save user's element */
+	scat->elements[scat->num_elements].buf = copy_buf;
+	scat->elements[scat->num_elements].len = (int) (curr_ptr - copy_buf);
+	++scat->num_elements;
+
+	if (i == num_recvrs)                                    /* all recvrs were legal above */
+	  ret = SP_multigroup_scat_multicast(mbox, serv_type, num_recvrs, 
+					     (const char(*)[MAX_GROUP_NAME]) recvrs, 
+					     mess_type, scat);
+	else
+	  ret = ILLEGAL_RECEIVERS;
+      }
+      if (fix_scat) {                       /* I modified user's scatter: I need to restore it */
+	--scat->num_elements;
+	if (ret >= 0) {          /* successful return: need to adjust err to ignore appendings */
+	  if (ret >= scat->elements[scat->num_elements].len)
+	    ret -= scat->elements[scat->num_elements].len;
+	  else
+	    stderr_abort("(%s, %d): mbox %d: serv 0x%X: group '%s': SP_multicast returned %d\n",
+			 __FILE__, __LINE__, mbox, serv_type, grp, ret);
+	}
+	scat->elements[scat->num_elements] = senders_elem;            /* restore modified elem */
+      }
+    }
+    unlock_conn(conn);                                                    /* UNLOCK CONNECTION */
+  } else {
+    DEBUG(std_stkfprintf(stderr, 0, "FL ILLEGAL_SESSION(%d)\n", mbox));
+    ret = ILLEGAL_SESSION; 
+  }
+  if (ret == CONNECTION_CLOSED || ret == ILLEGAL_SESSION)
+    FL_disconnect(mbox);
+
+  DEBUG(std_stkfprintf(stderr, -1, "FL_int_scat_multicast: ret %d, mbox %d\n", ret, mbox));
+  return ret;
+}
+
+/* try to read in a msg from the SP layer into m and make it presentable to the user 
+   returns non-zero if user's parameters shouldn't be processed, but, instead returned 
+*/
+static int FL_int_receive(gc_recv_mess *m) {
+  int alloced_groups = 0;                             /* boolean - did I alloc groups buffers? */
+  int alloced_buffer = 0;                           /* boolean - did I alloc a message buffer? */
+  int max_groups;                                        /* max_groups to be passed to receive */
+  int orig_max_groups = m->max_groups;                                    /* user's max_groups */
+  char (*groups)[MAX_GROUP_NAME] = m->groups;                /* groups to be passed to receive */
+  scatter *scat = m->scat_mess;                             /* scatter to be passed to receive */
+  int success = 0;                                                /* boolean - did it succeed? */
+
+  /* here I reserve space in groups for the destination group if it is a SUBGROUP_CAST msg */
+  if (m->max_groups != 0)
+    max_groups = m->max_groups - 1;
+  else
+    max_groups = 0;
+
+  /* don't use DROP_RECV semantics: flush needs to see entire data of msg: then we can drop */
+  *m->serv_type = (m->orig_serv_type & ~DROP_RECV);         
+
+  DEBUG(std_stkfprintf(stderr, 1, "FL_int_receive: mbox %d, serv 0x%X, max groups %d, "
+		       "grps %p, scat %p, scat_cap %ld\n", m->mbox, *m->serv_type, max_groups, 
+		       groups, scat, scat_capacity(scat)));
+
+  /* assert that there aren't any alloc'ed receive buffers, the user's 
+     max_groups was legal and m doesn't already have a msg */
+  assert(m->num_new_grps == 0 && m->new_msg.num_elements == 0 && 
+	 orig_max_groups >= 0 && !m->delivered);
+
+  /* perform the first receive */
+  m->ret = SP_scat_receive(m->mbox, m->serv_type, m->sender, max_groups, m->num_groups,
+			   groups, m->mess_type, m->endian_mismatch, scat);
+
+  /* check for buffer errors: should be if buffers too short, I didn't use DROP_RECV */
+  if (m->ret == GROUPS_TOO_SHORT || m->ret == BUFFER_TOO_SHORT) {
+    DEBUG(std_stkfprintf(stderr, 0, "Buffer error after initial recv: ret %d, num_groups %d, "
+			 "endian %d\n", m->ret, *m->num_groups, *m->endian_mismatch));
+
+    /* if the groups buffer was too short and the message was a SUBGROUP_CAST then the groups
+       buffer didn't have room for the destination group too: so decrement num_groups to reflect */
+    if (*m->num_groups < 0 && Is_subgroup_mess(*m->serv_type)) {
+      DEBUG(std_stkfprintf(stderr, 0, "GROUPS error, subgroup_mess: decrementing num_groups\n"));
+      --*m->num_groups;
+    }
+
+    /* if the user requested DROP_RECV, or if the user's groups was exactly big enough (because 
+       I shrunk max_groups above) and there was no msg buffer error (reported by endian_mismatch) 
+       or I can't tell if there was a buffer error because 3.12 didn't report that properly then
+       allocate the necessary buffers and re-receive - else return buffer error to user
+    */
+    if ((m->orig_serv_type & DROP_RECV) != 0 || 
+	(-*m->num_groups <= orig_max_groups && 
+	 (*m->endian_mismatch == 0 || FL_SP_version() == (float) 3.12))) {
+      DEBUG(std_stkfprintf(stderr, 0, "Either DROP_RECV or might be able to re-receive!\n"));
+
+      /* user's groups array actually was too small to contain the groups, so allocate some */
+      if (-*m->num_groups > orig_max_groups) {         /* implies *m->num_groups is negative */
+	size_t byte_size;
+
+	max_groups = -*m->num_groups;                     /* set max_groups to be big enough */
+	byte_size = max_groups * MAX_GROUP_NAME;          /* calculate necessary memory size */
+
+	DEBUG(std_stkfprintf(stderr, 0, "User's GROUPS buff was actually TOO SMALL %d < %d\n", 
+			     orig_max_groups, -*m->num_groups));
+
+	alloced_groups  = 1;                         /* record that I alloc'ed group buffers */
+	m->num_new_grps = max_groups;               /* record size of alloc'ed group buffers */
+
+	/* actually allocate a new groups array to be used in re-receive */
+	if ((groups = m->new_grps = (char(*)[MAX_GROUP_NAME]) malloc(byte_size)) == 0)
+	  stderr_abort("(%s, %d): malloc(%u)\n", __FILE__, __LINE__, byte_size);
+      } else {
+	DEBUG(std_stkfprintf(stderr, 0, "User's GROUPS buff was big enough %d >= %d\n",
+			     orig_max_groups, -*m->num_groups));
+	max_groups = orig_max_groups;        /* groups will fit into user's original buffers */
+      }
+      
+      /* now check and see if the user's msg buffer was too small: necessary size reported in
+         endian_mismatch in versions later than 3.12, 3.12 didn't do it but did do DROP_RECV */
+      if (*m->endian_mismatch < 0 || FL_SP_version() == (float) 3.12) {
+	DEBUG(std_stkfprintf(stderr, 0, "Endian mismatch reports msg buffer is too small\n"));
+	alloced_buffer = 1;                          /* record that I alloc'ed a mess buffer */
+	scat = &m->new_msg;                         /* point scat at my alloc'ed mess buffer */
+	scat->num_elements = 1;
+
+	/* figure out how big the mess buffer needs to be */
+	if (FL_SP_version() != (float) 3.12)
+	  scat->elements[0].len = -*m->endian_mismatch;       /* capacity of msg to be recvd */
+	else
+	  scat->elements[0].len = 102400;      /* endian_mismatch is buggy, use max msg size */
+
+	/* actually allocate the new mess buffer */
+	if ((scat->elements[0].buf = (char*) malloc(scat->elements[0].len)) == 0)
+	  stderr_abort("(%s, %d): malloc(%d)\n", __FILE__, __LINE__, scat->elements[0].len);
+      } else
+	DEBUG(std_stkfprintf(stderr, 0, "No msg buffer error reported\n"));
+
+      /* try to receive again (shouldn't fail now) and don't use DROP_RECV again */
+      *m->serv_type = (m->orig_serv_type & ~DROP_RECV);
+      DEBUG(std_stkfprintf(stderr, 0, "Re-receive: mbox %d, serv 0x%X, max groups %d, grps %p, "
+			   "scat %p, scat_cap %ld\n", m->mbox, *m->serv_type, max_groups,
+			   groups, scat, scat_capacity(scat)));
+
+      /* perform the re-receive */
+      m->ret = SP_scat_receive(m->mbox, m->serv_type, m->sender, max_groups, m->num_groups,
+			       groups, m->mess_type, m->endian_mismatch, scat);
+
+      /* if we had a buffer problem then code above or SP is buggy -> abort */
+      if (m->ret == GROUPS_TOO_SHORT || m->ret == BUFFER_TOO_SHORT)
+	stderr_abort("(%s, %d): mbox %d: buggy SP_recv DROP_RECV ret %d: "
+		     "max_groups %d: num_groups %d: scat_cap %ld: endian_mismatch %d\n", 
+		     __FILE__, __LINE__, m->mbox, m->ret, max_groups, *m->num_groups, 
+		     scat_capacity(scat), *m->endian_mismatch);
+    }
+  }
+  /* receive or re-receive succeeded */
+  if (m->ret >= 0) {
+    int not_end = 1;                            /* boolean - did I not seek to end of msg yet? */
+    scatp pos, user_pos;
+    long err;
+
+    /* successful recv: debug print out what I got! */
+    DEBUG(std_stkfprintf(stderr, 0, "Successful recv: ret %d, serv 0x%X, sender '%s', "
+			 "num_groups %d, mess_type %d, endian %d, Groups:\n", m->ret, 
+			 *m->serv_type, m->sender, *m->num_groups, 
+			 *m->mess_type, *m->endian_mismatch);
+	  for (err = 0; err < *m->num_groups; ++err) {
+	    std_stkfprintf(stderr, 0, "\t`%s'\n", groups[err]);
+	  });
+
+    /* subgroup and flush messages have appended data that needs to be extracted and removed */
+    if (Is_regular_mess(*m->serv_type)) {
+      if (Is_subgroup_mess(*m->serv_type)) {  /* subgroup msgs have destination group appended */
+	char *ref_grp;
+
+	assert(m->ret >= MAX_GROUP_NAME);                                /* ensure fully recvd */
+	m->ret -= MAX_GROUP_NAME;                           /* correct to ignore the appendage */
+	
+	err = scatp_set(&pos, scat, m->ret, SEEK_SET);                /* seek to copy position */
+	assert(err == 0);
+	not_end = 0;                                       /* pos was seeked to new end of msg */
+	
+	/* copy out reference group: put after last used element in groups: should fit */
+	ref_grp = (char*) (groups + *m->num_groups);
+	err = scatp_cpy1(ref_grp, &pos, MAX_GROUP_NAME);
+	assert(err == MAX_GROUP_NAME);
+	++*m->num_groups;                 /* increment num_groups to include destination group */
+	DEBUG(std_stkfprintf(stderr, 0, "Subgroupcast mess: Ref group is '%s'!\n", ref_grp));
+      }
+
+      /* all reserved FLUSH msg types have a group_id attached on end of msg */
+      if (IS_ILLEGAL_SEND_MTYPE(*m->mess_type)) {
+	DEBUG(std_stkfprintf(stderr, 0, "Recvd an internal flush mess of type %s\n", 
+			     *m->mess_type == FLUSH_OK_MESS ? "FLUSH_OK_MESS" : 
+			     (*m->mess_type == FLUSH_RECV_MESS ? "FLUSH_RECV_MESS" : 
+			      (*m->mess_type == VULNERABLE_MESS ? "VULNERABLE_MESS" : 
+			       "UNKNOWN TYPE!"))));
+	assert(m->ret >= sizeof(group_id));                              /* ensure fully recvd */
+	m->ret -= sizeof(group_id);                         /* correct to ignore the appendage */
+	
+	if (not_end) {
+	  err = scatp_set(&pos, scat, m->ret, SEEK_SET);              /* seek to copy position */
+	  assert(err == 0);
+	  not_end = 0;                                                        /* seeked to end */
+	} else {
+	  err = scatp_jbackward(&pos, sizeof(group_id));                 /* move back from end */
+	  assert(err == sizeof(group_id));
+	}
+	err = scatp_cpy1((char*) &m->dst_gid, &pos, sizeof(group_id));         /* copy out vid */
+	assert(err == sizeof(group_id));
+
+	/* endian correct the vid if necessary */
+	if (*m->endian_mismatch != 0) {
+	  stdflip32(m->dst_gid.id);
+	  stdflip32(m->dst_gid.id + 1);
+	  stdflip32(m->dst_gid.id + 2);
+	}
+	DEBUG(std_stkfprintf(stderr, 0, "Dst gid is %d %d %d!\n", m->dst_gid.id[0], 
+			     m->dst_gid.id[1], m->dst_gid.id[2]));
+
+	/* VULNERABLE msgs also have the original user's msg type appended */
+	if (*m->mess_type == VULNERABLE_MESS) { 
+	  assert(m->ret >= sizeof(int16));                               /* ensure fully recvd */
+	  m->ret -= sizeof(int16);                          /* correct to ignore the appendage */
+	  
+	  m->vulnerable = 1;                                           /* mark m as vulnerable */
+
+	  /* here should already be seeked to end */
+	  err = scatp_jbackward(&pos, sizeof(int16));              /* move back from end again */
+	  assert(err == sizeof(int16));	  
+	  err = scatp_cpy1((char*) m->mess_type, &pos, sizeof(int16));    /* cpy out mess type */
+	  assert(err == sizeof(int16));
+	  
+	  /* endian correct the message type if necessary */
+	  if (*m->endian_mismatch != 0)
+	    stdflip16(m->mess_type);
+
+	  DEBUG(std_stkfprintf(stderr, 0, "VULNERABLE: Orig msg type is %d!\n", *m->mess_type));
+	}
+      }
+    } else if (Is_reg_memb_mess(*m->serv_type)) {
+      /* if its a regular membership message read the group id out of the msg */
+      assert(m->ret >= sizeof(group_id) + sizeof(int) + MAX_GROUP_NAME);    /* ensure received */
+      err = scatp_begin(&pos, scat);
+      assert(err == 0);
+      err = scatp_cpy1((char*) &m->dst_gid, &pos, sizeof(group_id));       /* copy out new gid */
+      assert(err == sizeof(group_id));                             /* already endian corrected */
+      DEBUG(std_stkfprintf(stderr, 0, "Recvd a SP reg memb mess: New SP gid is %d %d %d\n", 
+			   m->dst_gid.id[0], m->dst_gid.id[1], m->dst_gid.id[2]));
+    }
+
+    /* if I alloc'ed a msg buffer -> user probably requested DROP_RECV, need to copy from 
+       alloc'ed buffers into user's buffers and set m->ret to an appropriate value */
+    if (alloced_buffer) {
+      m->new_msg.elements[0].len = m->ret;                          /* record size of user msg */
+      err = scatp_begin(&user_pos, m->scat_mess);
+      assert(err == 0);
+      err = scatp_begin(&pos, &m->new_msg);
+      assert(err == 0);
+
+      /* perform copy from alloc'ed scat to user's scat */
+      err = scatp_cpy0(&user_pos, &pos, m->ret); 
+      assert(err >= 0 && err <= m->ret);                 /* shouldn't detect illegal scat here */
+
+      /* possible that removing appended data made the user's msg buffer big enough */
+      /* this also catches if I didn't really need to allocate a msg buffer (SP 3.12 bug) */
+      if (err != m->ret) {                              /* didn't all fit into user's msg buff */
+	DEBUG(std_stkfprintf(stderr, 0, "User msg buffer still too short!\n"));
+
+	/* 3.12: if !DROP_RECV, then report negative msg size correctly in endian_mismatch */
+	if ((m->orig_serv_type & DROP_RECV) == 0 && FL_SP_version() == (float) 3.12) {
+	  DEBUG(std_stkfprintf(stderr, 0, "Alloced msg buff: SP 3.12 DROP_RECV bug: error!\n"));
+	  success = -1;    /* non-zero success indicates an error that should be returned now! */
+	  *m->endian_mismatch = -m->ret;
+	}
+	m->ret = BUFFER_TOO_SHORT;
+      } else /* their buffer was big enough! */
+	DEBUG(std_stkfprintf(stderr, 0, "Unnecessary msg buff alloc! No BUFFER_TOO_SHORT!\n"));
+    }
+
+    /* if I alloc'ed groups buffers then user's buffers were too small and requested DROP_RECV */
+    if (alloced_groups) {
+      DEBUG(std_stkfprintf(stderr, 0, "Alloced groups buff, copying over: GROUPS_TOO_SHORT\n"));
+      memcpy(m->groups, m->new_grps, m->max_groups * MAX_GROUP_NAME);
+
+      /* if it is a subgroup msg and there is space put the destination group at end of groups */
+      if (Is_subgroup_mess(*m->serv_type) && m->max_groups > 0)
+	memcpy(m->groups[m->max_groups - 1], m->new_grps[*m->num_groups - 1], MAX_GROUP_NAME);
+
+      *m->num_groups = -*m->num_groups;                                    /* report too short */
+      m->ret = GROUPS_TOO_SHORT;
+    }
+  } else {
+    DEBUG(std_stkfprintf(stderr, 0, "Error from recv or re-recv that needs to be returned\n"));
+    success = -1;     /* non-zero success indicates an error that should be given to user now! */
+  }
+  DEBUG(std_stkfprintf(stderr, -1, "SP_int_receive: success %d, ret %d, mbox %d, serv 0x%X, "
+		       "mess_type %d\n", success, m->ret, m->mbox, *m->serv_type, 
+		       *m->mess_type));
+  return success;
+}
+
+/* state machine code */
+/* recv_and_handle: main director function: returns whether or not still have conn lock */
+static int recv_and_handle(fl_conn *conn, gc_recv_mess *m) { 
+  fl_group *group;
+  int ret = 1;                                           /* do I still have the conn lock? */
+
+  DEBUG(std_stkfprintf(stderr, 1, "recv_and_handle: mbox(%d, %p)\n", conn->mbox, conn));
+  release_conn_lock(conn);    /* let go of conn lock so I don't block sends on this connection */
+  
+  if (FL_int_receive(m) != 0) {           /* if an immediate return recv error: return to user */
+    DEBUG(std_stkfprintf(stderr, 0, "immediate return from FL_int_receive %d\n", m->ret));
+    m->delivered = 1;
+    ret = 0;
+  } else if (!acquire_conn_lock(conn)) {        /* couldn't acquire conn lock -> disconnecting */
+    DEBUG(std_stkfprintf(stderr, 0, "Couldn't reacquire conn lock -> disconnecting\n"));
+    m->ret = ILLEGAL_SESSION;
+    m->delivered = 1;
+    ret = 0;
+  } else if (Is_regular_mess(*m->serv_type)) {                           /* SP regular message */
+    char (*dst_grp)[MAX_GROUP_NAME];                     /* group this message is intended for */
+    int num_groups;
+
+    /* for regular msgs dst group is the first group, for subgroup msgs it is the last group */
+    get_groups_info(m, &num_groups, &dst_grp);
+    if (Is_subgroup_mess(*m->serv_type))
+      dst_grp += num_groups - 1;
+    
+    DEBUG(std_stkfprintf(stderr, 0, "A regular mess destined for group '%s'\n", dst_grp));
+    if ((group = get_group(conn, (char*) dst_grp)) != 0) {     /* look up fl_group structure */
+      if (*m->mess_type == FLUSH_OK_MESS)
+	handle_recv_flush_ok(conn, group, m);
+      else if (*m->mess_type == FLUSH_RECV_MESS)
+	handle_recv_flush_recv(conn, group, m);
+      else
+	handle_recv_reg_mess(conn, group, m);
+    } else if (strncmp((char*) dst_grp, conn->private, MAX_GROUP_NAME) == 0) {  /* my pgroup */
+      DEBUG(std_stkfprintf(stderr, 0, "Mess is addressed to my private group! Deliver!\n"));
+      deliver(conn, m, 0, 0);                                 /* deliver msg immediately */
+    } else
+      stderr_msg("(%s, %d): WARNING: a regular msg for a group of which I'm not a member!\n"
+		 "mbox %d: serv 0x%X: sender '%s': dst_grp '%s': mess_type %d\n", __FILE__, 
+		 __LINE__, m->mbox, *m->serv_type, m->sender, dst_grp, *m->mess_type);
+  } else if (Is_membership_mess(*m->serv_type)) {                   /* SP membership message */
+    group = get_group(conn, m->sender);       /* look up fl_group structure: might not exist */
+
+    DEBUG(std_stkfprintf(stderr, 0, "A memb mess destined for group ('%s', %p)\n", 
+			 m->sender, group));
+    if (Is_reg_memb_mess(*m->serv_type))
+      handle_recv_reg_memb_mess(conn, group, m);
+    else if (Is_transition_mess(*m->serv_type))
+      handle_recv_trans_memb_mess(conn, group, m);
+    else if (Is_caused_leave_mess(*m->serv_type))
+      handle_recv_self_leave_memb_mess(conn, group, m);
+    else
+      stderr_abort("(%s, %d): mbox %d: recv_and_handle: unexpected membership 0x%X: "
+		   "group '%s'\n", __FILE__, __LINE__, conn->mbox, *m->serv_type, m->sender);
+  } else
+    stderr_abort("(%s, %d): mbox %d: recv_and_handle: unexpected service 0x%X: "
+		 "sender '%s'\n", __FILE__, __LINE__, conn->mbox, *m->serv_type, m->sender);
+
+  DEBUG(std_stkfprintf(stderr, -1, "recv_and_handle: ret %d (have_conn_lock)\n", (int) ret));
+  return ret;
+}
+
+static void handle_recv_flush_ok(fl_conn *conn, fl_group *group, gc_recv_mess *m) {
+  sp_memb_change *spc;
+  stdit hit;
+  stdit lit;
+
+  DEBUG(std_stkfprintf(stderr, 1, "handle_recv_flush_ok: mbox(%d, %p), group('%s', %p), %s\n",
+		       conn->mbox, conn, group->group, group, state_str(group->vstate)));
+
+  /* see if I know about this flok's gid yet or not: if not create a sp_memb_change */
+  if (!stdhash_is_end(&group->pmemb_hash, stdhash_find(&group->pmemb_hash, &hit, &m->dst_gid))) {
+    spc = *(sp_memb_change**) stdhash_it_val(&hit);
+    DEBUG(std_stkfprintf(stderr, 0, "Recvd a flok for a known gid %d %d %d: spc %p\n", 
+			 m->dst_gid.id[0], m->dst_gid.id[1], m->dst_gid.id[2], spc));
+    assert(SP_equal_group_ids(m->dst_gid, spc->memb_info->gid));
+  } else {
+    spc = create_sp_memb_change(m->dst_gid);
+    DEBUG(std_stkfprintf(stderr, 0, "Recvd a flok for an UNKNWON gid %d %d %d: CREATED spc %p\n",
+			 m->dst_gid.id[0], m->dst_gid.id[1], m->dst_gid.id[2], spc));
+    stdhash_insert(&group->pmemb_hash, 0, &m->dst_gid, &spc);           /* put into pmemb_hash */
+  }
+  stdhash_insert(&spc->flok_senders, 0, m->sender, 0);              /* record flok from sender */
+  DEBUG(std_stkfprintf(stderr, 0, "curr_change(%p) ?= spc(%p) now has %lu floks, needs %d\n", 
+		       group->curr_change, spc, stdhash_size(&spc->flok_senders), 
+		       spc->memb_mess_recvd ? spc->memb_info->orig_num_membs : -1));
+  switch (group->vstate) {
+  case AGREE:
+    assert(group->curr_change != 0 && group->curr_change->memb_mess_recvd);
+    if (spc == group->curr_change &&
+	stdhash_size(&spc->flok_senders) == spc->memb_info->orig_num_membs) {
+      DEBUG(std_stkfprintf(stderr, 0, "Recvd last needed flok for curr change! INSTALL!\n"));
+      /* Received all the floks necessary to install the next fl view! */
+      install_new_view(conn, group, m);            /* deliver an appropriate fl membership msg */
+
+      DEBUG(std_stkfprintf(stderr, 0, "Going to state VERIFY!\n"));
+      group->vstate = VERIFY;                                            /* update group state */
+      if (group->mstate == JOINING)                                  
+	group->mstate = JOINED;                               /* has been installed in a group */
+
+      update_fl_view(group);                     /* update the view information for this group */
+
+      /* if no cascading membership: send a FLUSH_RECV, else deliver a FLUSH_REQ mess */
+      if (stddll_empty(&group->memb_queue)) {
+	int err;
+
+	DEBUG(std_stkfprintf(stderr, 0, "No cascading memberships -> sending FLUSH_RECV\n"));
+	err = SP_multicast(conn->mbox, FIFO_MESS, group->group, FLUSH_RECV_MESS,
+			   sizeof(group_id), (char*) &group->fl_view->gid);
+
+	if (err == CONNECTION_CLOSED || err == ILLEGAL_SESSION) {    /* immediate return error */
+	  DEBUG(std_stkfprintf(stderr, 0, "SP_multicast failure %ld\n", err));
+	  m->delivered = 1;                                   /* return to user immediately */
+	  m->ret = (int) err;
+	  break;
+	} else if (err != sizeof(group_id))
+	  stderr_abort("(%s, %d): SP_multicast unexpected return %d\n", __FILE__, __LINE__, err);
+      } else {                            /* buffer a FLUSH_REQ mess, update curr_change, etc. */
+	DEBUG(std_stkfprintf(stderr, 0, "Cascading Memberships(%lu) to handle!\n", 
+			     stddll_size(&group->memb_queue)));
+	handle_next_memb_change(conn, group, m);
+      }
+      /* if new members have already died (after they sent their floks): deliver TRANS */
+      if (stdhash_size(&group->fl_view->curr_membs) < group->fl_view->orig_num_membs) {
+	DEBUG(std_stkfprintf(stderr, 0, "New view members have already died! Deliver TRANS!\n"));
+	assert(!group->fl_view->in_trans_memb);
+	group->fl_view->in_trans_memb = 1;
+	deliver_trans_sig(conn, m, group->group);                               /* will buffer */
+      }
+      /* deliver any VULNERABLE messages that were pre-delivered */
+      stddll_begin(&group->mess_queue, &lit);
+      for (; !stddll_is_end(&group->mess_queue, &lit); stddll_it_next(&lit)) {
+	DEBUG(std_stkfprintf(stderr, 0, "Delivering a vulnerable SP pre-delivered msg!\n"));
+	deliver(conn, m, *(gc_buff_mess**) stddll_it_val(&lit), 1);          /* will buffer */
+      }
+      stddll_clear(&group->mess_queue);                    /* empty postponed vulnerable queue */
+    } else
+      assert(!spc->memb_mess_recvd || 
+	     stdhash_size(&spc->flok_senders) < spc->memb_info->orig_num_membs);
+    break;
+  case AUTHORIZE: 
+    assert(group->curr_change != 0 && group->curr_change->memb_mess_recvd);
+    assert(!spc->memb_mess_recvd || 
+	   stdhash_size(&spc->flok_senders) < spc->memb_info->orig_num_membs);
+    break;
+  case STEADY: case VERIFY: 
+    assert(group->curr_change == 0);
+    assert(!spc->memb_mess_recvd || 
+	   stdhash_size(&spc->flok_senders) < spc->memb_info->orig_num_membs);
+    break;
+  default: stderr_abort("(%s, %d): impossible vstate %d\n", __FILE__, __LINE__, group->vstate);
+  }
+  DEBUG(std_stkfprintf(stderr, -1, "handle_recv_flush_ok: mbox(%d, %p), group('%s', %p), %s\n",
+		       conn->mbox, conn, group->group, group, state_str(group->vstate)));
+}
+
+static void handle_recv_flush_recv(fl_conn *conn, fl_group *group, gc_recv_mess *m) {
+  DEBUG(std_stkfprintf(stderr, 1, "handle_recv_flush_recv: mbox(%d, %p), group('%s', %p), %s\n",
+		       conn->mbox, conn, group->group, group, state_str(group->vstate)));
+  switch (group->vstate) {
+  case VERIFY: 
+    assert(group->curr_change == 0);
+    if (SP_equal_group_ids(m->dst_gid, group->fl_view->gid)) {
+      DEBUG(std_stkfprintf(stderr, 0, "Got a FLUSH_RECV for curr FL vid need %d flush_recvs!\n", 
+			   group->fl_view->orig_num_membs));
+      if (++group->flush_recvs == group->fl_view->orig_num_membs) {
+	DEBUG(std_stkfprintf(stderr, 0, "Got last FLUSH_RECV: going to state STEADY!\n"));
+	group->vstate = STEADY;
+      }
+    } else
+      DEBUG(std_stkfprintf(stderr, 0, "Ignoring FLUSH_RECV for vid %d %d %d while in VERIFY!\n",
+			   m->dst_gid.id[0], m->dst_gid.id[1], m->dst_gid.id[2]));
+    break;
+  case AGREE:
+    assert(group->curr_change != 0 && group->curr_change->memb_mess_recvd);
+    if (SP_equal_group_ids(m->dst_gid, group->curr_change->memb_info->gid)) {
+      DEBUG(std_stkfprintf(stderr, 0, "Got a FLUSH_RECV for CURR CHANGE need %d flush_recvs!\n", 
+			   group->fl_view->orig_num_membs));
+      ++group->flush_recvs;
+      assert(group->flush_recvs < group->curr_change->memb_info->orig_num_membs);
+    } else 
+      DEBUG(std_stkfprintf(stderr, 0, "Ignoring FLUSH_RECV for vid %d %d %d while in AGREE!\n",
+			   m->dst_gid.id[0], m->dst_gid.id[1], m->dst_gid.id[2]));
+    break;
+  case STEADY: 
+    assert(group->curr_change == 0);
+    DEBUG(std_stkfprintf(stderr, 0, "Ignoring FLUSH_RECV for vid %d %d %d while in STEADY!\n",
+			 m->dst_gid.id[0], m->dst_gid.id[1], m->dst_gid.id[2]));
+    break;
+  case AUTHORIZE: 
+    assert(group->curr_change != 0 && group->curr_change->memb_mess_recvd);
+    DEBUG(std_stkfprintf(stderr, 0, "Ignoring FLUSH_RECV for vid %d %d %d while in AUTHORIZE!\n",
+			 m->dst_gid.id[0], m->dst_gid.id[1], m->dst_gid.id[2]));
+    break;
+  default: stderr_abort("(%s, %d): impossible vstate %d\n", __FILE__, __LINE__, group->vstate);
+  }
+  DEBUG(std_stkfprintf(stderr, -1, "handle_recv_flush_recv: mbox(%d, %p), group('%s', %p), %s\n",
+		       conn->mbox, conn, group->group, group, state_str(group->vstate)));
+}
+
+static void handle_recv_reg_mess(fl_conn *conn, fl_group *group, gc_recv_mess *um) {
+  stdit hit;
+  gc_buff_mess *bm;
+
+  DEBUG(std_stkfprintf(stderr, 1, "handle_recv_reg_mess: mbox(%d, %p), group('%s', %p), %s\n",
+		       conn->mbox, conn, group->group, group, state_str(group->vstate)));
+  switch (group->vstate) {
+  case AGREE:
+    assert(group->curr_change != 0 && group->curr_change->memb_mess_recvd);
+    /* if vulnerable message meant for the next membership from a member of the next one */
+    if (um->vulnerable) {
+      DEBUG(std_stkfprintf(stderr, 0, "Recvd a vulnerable msg!\n"));
+      if (SP_equal_group_ids(um->dst_gid, group->curr_change->memb_info->gid)) {
+	DEBUG(std_stkfprintf(stderr, 0, "Addressed to my next FL vid!\n"));
+	if (!stdhash_is_end(&group->curr_change->memb_info->curr_membs, stdhash_find(&group->curr_change->memb_info->curr_membs, 
+					    &hit, &um->sender))) {
+	  DEBUG(std_stkfprintf(stderr, 0, "Recvd a vuln msg that needs to be POSTPONED!\n"));
+	  if ((bm = (gc_buff_mess*) malloc(sizeof(gc_buff_mess))) == 0)
+	    stderr_abort("(%s, %d): malloc(%u)\n", __FILE__, __LINE__, sizeof(gc_buff_mess));
+	  
+	  userm_to_buffm(bm, um);
+	  stddll_push_back(&group->mess_queue, &bm);
+	  break;
+	} else
+	  stderr_abort("'%s' Not in my next FL curr membs!\n", um->sender);
+      } else
+	DEBUG(std_stkfprintf(stderr, 0, "Vuln msg not to next FL vid: should ignore below\n"));
+    } /* ELSE FALL THROUGH TO NORMAL CASES, BELOW (other case statements)!!!! */
+  case STEADY: case AUTHORIZE: case VERIFY:
+    /* if the message is from a current flush member then deliver it */
+    if (!stdhash_is_end(&group->fl_view->curr_membs, stdhash_find(&group->fl_view->curr_membs, &hit, &um->sender))) {
+      DEBUG(std_stkfprintf(stderr, 0, "Deliver reg mess from group memb '%s'\n", um->sender));
+      deliver(conn, um, 0, 0);
+    } else
+      DEBUG(std_stkfprintf(stderr, 0, "Ignore reg mess from non group memb '%s'\n", um->sender));
+    break;
+  default: stderr_abort("(%s, %d): impossible vstate %d\n", __FILE__, __LINE__, group->vstate);
+  }
+  DEBUG(std_stkfprintf(stderr, -1, "handle_recv_reg_mess: mbox(%d, %p), group('%s', %p), %s\n",
+		       conn->mbox, conn, group->group, group, state_str(group->vstate)));
+}
+
+static void 
+handle_recv_reg_memb_mess(fl_conn *conn, fl_group *group, gc_recv_mess *m) {
+  sp_memb_change *spc, *new_curr_change;
+  stdhash leavers;               /* leavers<char*, nil>: members that left since last SP memb  */
+  stdit hit, flit;
+  stdit lit;
+  char (*groups)[MAX_GROUP_NAME];
+  int num_groups;
+  int fl_memb_shrunk = 0;
+
+  DEBUG(std_stkfprintf(stderr, 1, "handle_recv_reg_memb_mess: mbox(%d, %p), group('%s', %p), %s"
+		       ", gid %d %d %d\n", conn->mbox, conn, m->sender, group, 
+		       group != 0 ? state_str(group->vstate) : "NO STATE", m->dst_gid.id[0],
+		       m->dst_gid.id[1], m->dst_gid.id[2]));
+
+  if (group == 0) {     /* no fl_group struct for this group yet -> create one and put in conn */
+    group = add_group(conn, m->sender);
+    DEBUG(std_stkfprintf(stderr, 0, "No group structure yet, CREATED group %p!\n", group));
+  }
+  /* if a sp_memb_change for this gid doesn't exist yet: create one and put in pmemb */
+  if (stdhash_is_end(&group->pmemb_hash, stdhash_find(&group->pmemb_hash, &hit, &m->dst_gid))) {
+    spc = create_sp_memb_change(m->dst_gid);
+    DEBUG(std_stkfprintf(stderr, 0, "Unknown gid: CREATED spc %p\n", spc));
+    stdhash_insert(&group->pmemb_hash, 0, &m->dst_gid, &spc);
+  } else {
+    spc = *(sp_memb_change**) stdhash_it_val(&hit);
+    DEBUG(std_stkfprintf(stderr, 0, "KNOWN gid: got from pmemb_hash %p\n", spc));
+  }
+  get_groups_info(m, &num_groups, &groups);                    /* fill in rest of data for spc */
+  assert(!spc->memb_mess_recvd);
+  spc->memb_mess_recvd = 1;                             /* I got the memb mess for this gid */
+  spc->delta_member = determine_leavers(&leavers, group->sp_view, m); /* who left from last SP */
+  fill_view(spc->memb_info, *m->serv_type, num_groups, groups, *m->mess_type);     /* spc view */
+
+  /* update the current fl_view: shrink the fl membership by removing any leavers */
+  for (stdhash_begin(&leavers, &hit); !stdhash_is_end(&leavers, &hit); stdhash_it_next(&hit)) {
+    if (!stdhash_is_end(&group->fl_view->curr_membs, stdhash_find(&group->fl_view->curr_membs, 
+					&flit, stdhash_it_key(&hit)))) {
+      DEBUG(std_stkfprintf(stderr, 0, "FL member '%s' left in memb event, remove from fl view\n",
+			   *(char**) stdhash_it_key(&hit)));
+      fl_memb_shrunk = 1;                                 /* remember the membership shrunk */
+      stdhash_erase(&group->fl_view->curr_membs, &flit);                              /* remove from fl_view->curr_membs */
+    }
+  }
+  /* remove any memberships that this SP memb change invalidated */
+  age_and_invalidate_pmembs(group);            /* age unknown spc's and remove them if too old */
+  new_curr_change = collapse_memberships(group, &leavers, spc);   /* call before pushing spc!! */
+  stdhash_destruct(&leavers);       /* after collapse members, leavers values could be invalid */
+
+  /* push spc on to end of memb queue and point sp view at spc's view */
+  /* spc must be pushed on to end of memb_queue after collapse memberships due to leave prob   */
+  stddll_push_back(&group->memb_queue, &spc); /* push this SP memb change on to the memb queue */
+  group->sp_view = spc->memb_info;                    /* sp_view refers to most recent SP view */
+
+  /* if I wasn't working on a SP memb change or I just invalidated it -> update curr_change */
+  /* this will deliver a FLUSH_REQ message if I wasn't already working on a change */
+  if (group->curr_change != new_curr_change) {
+    DEBUG(std_stkfprintf(stderr, 0, "New SP memb change to work on now: %p -> %p!!\n", 
+			 group->curr_change, new_curr_change));
+
+    /* drop any pre-delivered messages as they must have come from partioned members */
+    for (stddll_begin(&group->mess_queue, &lit); !stddll_is_end(&group->mess_queue, &lit); stddll_it_next(&lit)) {
+      DEBUG(std_stkfprintf(stderr, 0, "Destroying pre-delivered msg from leaver member\n"));
+      free_gc_buff_mess(*(gc_buff_mess**) stddll_it_val(&lit));
+    }
+    stddll_clear(&group->mess_queue);
+    handle_next_memb_change(conn, group, m);                   /* possibly deliver a FLUSH_REQ */
+  }
+  /* if the group shrunk and I haven't delivered a trans yet -> deliver a trans signal */
+  if (!group->fl_view->in_trans_memb && fl_memb_shrunk) {
+    DEBUG(std_stkfprintf(stderr, 0, "FL memb shrunk, haven't delivered TRANS: deliver TRANS\n"));
+    group->fl_view->in_trans_memb = 1;
+    deliver_trans_sig(conn, m, group->group);
+  }
+  DEBUG(std_stkfprintf(stderr, -1, "handle_recv_reg_memb mess mbox(%d, %p), group('%s', %p), "
+		       "%s\n", conn->mbox, conn, group->group, group, state_str(group->vstate)));
+}
+
+static void handle_next_memb_change(fl_conn *conn, fl_group *group, gc_recv_mess *m) {
+  stdit lit;
+  int err;
+
+  DEBUG(std_stkfprintf(stderr, 1, "handle_next_memb_change: mbox(%d, %p), group('%s', %p), "
+		       "%s\n", conn->mbox, conn, group->group, group, state_str(group->vstate)));
+
+  assert(!stddll_empty(&group->memb_queue));
+  /* special case for JOINING: don't have a curr change yet! */
+  assert((group->vstate == STEADY || group->vstate == VERIFY) ? 
+	 group->curr_change == 0 : group->mstate == JOINING || group->curr_change != 0);
+
+  group->curr_change = *(sp_memb_change**) stddll_it_val(stddll_begin(&group->memb_queue, &lit));
+  assert(group->curr_change != 0 && group->curr_change->memb_mess_recvd);
+  group->flush_recvs = 0;
+
+  switch (group->vstate) {
+  case STEADY: case VERIFY:
+    DEBUG(std_stkfprintf(stderr, 0, "Going to state AUTHORIZE!\n"));
+    group->vstate = AUTHORIZE;
+    deliver_flush_req(conn, m, group->group);
+    break;
+  case AUTHORIZE:            /* already waiting for a response from user */
+    break;
+  case AGREE:
+    DEBUG(std_stkfprintf(stderr, 0, "Going to state AUTHORIZE!\n"));
+    group->vstate = AUTHORIZE;
+    DEBUG(std_stkfprintf(stderr, 0, "AGREE: already auth'ed memb, send flok to new gid!\n"));
+    if ((err = FL_int_flush(conn, group)) != 0) { /* unrecoverable error */
+      DEBUG(std_stkfprintf(stderr, 0, "FL_int flush error: return to user\n"));
+      m->ret = err;
+      m->delivered = 1;
+    }
+    break;
+  default: stderr_abort("(%s, %d): impossible vstate %d\n", __FILE__, __LINE__, group->vstate);
+  }
+  DEBUG(std_stkfprintf(stderr, -1, "handle_next_memb_change: mbox(%d, %p), group('%s', %p), "
+		       "%s\n", conn->mbox, conn, group->group, group, state_str(group->vstate)));
+}
+
+static void 
+handle_recv_trans_memb_mess(fl_conn *conn, fl_group *group, gc_recv_mess *m) {    
+  DEBUG(std_stkfprintf(stderr, 1, "handle_recv_trans_memb_mess: mbox(%d, %p), group('%s', %p), "
+		       "%s\n", conn->mbox, conn, group->group, group, state_str(group->vstate)));
+  group->sp_view->in_trans_memb = 1;
+  if (!group->fl_view->in_trans_memb) {
+    DEBUG(std_stkfprintf(stderr, 0, "Got a TRANS for first time in FL view: deliver TRANS\n"));
+    group->fl_view->in_trans_memb = 1;
+    deliver_trans_sig(conn, m, group->group);
+  }
+  DEBUG(std_stkfprintf(stderr, -1, "handle_recv_trans_memb_mess: mbox(%d, %p), group('%s', %p), "
+		       "%s\n", conn->mbox, conn, group->group, group, state_str(group->vstate)));
+}
+
+static void 
+handle_recv_self_leave_memb_mess(fl_conn *conn, fl_group *group, gc_recv_mess *m){
+  DEBUG(std_stkfprintf(stderr, 1, "handle_recv_self_leave_memb_mess: mbox(%d, %p), "
+		       "group('%s', %p), %s\n", conn->mbox, conn, group->group, group, 
+		       state_str(group->vstate)));
+  remove_group(conn, group);
+  deliver(conn, m, 0, 0);
+  DEBUG(std_stkfprintf(stderr, -1, "handle_recv_self_leave_memb_mess: mbox(%d, %p)\n",
+		       conn->mbox, conn));
+}
+
+/* install the current view I was trying to finish */
+static void install_new_view(fl_conn *conn, const fl_group *group, gc_recv_mess *m) {
+  view *new_fl_view = group->curr_change->memb_info, *last_fl_view = group->fl_view;
+  int not_network = !Is_caused_network_mess(new_fl_view->memb_type);
+  char (*curr_vs)[MAX_GROUP_NAME];
+  gc_buff_mess *memb_mess;
+  size_t size, byte_size;
+
+  DEBUG(std_stkfprintf(stderr, 1, "install_new_view: mbox(%d, %p), group('%s', %p), %s\n",
+		       conn->mbox, conn, group->group, group, state_str(group->vstate)));
+
+  /* I create a gc_buff_mess and then deliver it to the user for to simplify msg construction */
+  if ((memb_mess = (gc_buff_mess*) malloc(sizeof(gc_buff_mess))) == 0)
+    stderr_abort("(%s, %d): malloc(%u)\n", sizeof(gc_buff_mess));
+      
+  memb_mess->mbox = conn->mbox;              /* set mbox, service type, group, and num_groups */
+  memb_mess->serv_type = new_fl_view->memb_type;
+  assert(Is_reg_memb_mess(memb_mess->serv_type));
+  strncpy(memb_mess->sender, group->group, MAX_GROUP_NAME);
+  memb_mess->num_groups = new_fl_view->orig_num_membs;
+      
+  byte_size = memb_mess->num_groups * MAX_GROUP_NAME;             /* fill in the groups array */
+  if ((memb_mess->groups = (char(*)[MAX_GROUP_NAME]) malloc(byte_size)) == 0)
+    stderr_abort("(%s, %d): malloc(%u)\n", byte_size);
+  memcpy(memb_mess->groups, new_fl_view->membs_names, byte_size);
+
+  memb_mess->mess_type = new_fl_view->my_index;           /* set mess_type and endian_mismatch */
+  memb_mess->endian_mismatch = 0;
+
+  if (not_network) {                                       /* set the mess_len (return val) */
+    size = 1;
+    memb_mess->mess_len = sizeof(group_id) + sizeof(int) + MAX_GROUP_NAME;
+  } else {
+    size = stdhash_size(&last_fl_view->curr_membs);
+    byte_size = size * MAX_GROUP_NAME;
+    memb_mess->mess_len = sizeof(group_id) + sizeof(int) + byte_size;
+  }
+  if ((memb_mess->mess = (char*) malloc(memb_mess->mess_len)) == 0) /* allocate body of msg */
+    stderr_abort("(%s, %d): malloc(%u)\n", byte_size);
+      
+  memcpy(memb_mess->mess, &new_fl_view->gid, sizeof(group_id));   /* fill in body: gid, num_vs */
+  memcpy(memb_mess->mess + sizeof(group_id), &size, sizeof(int));
+  curr_vs = (char(*)[MAX_GROUP_NAME]) (memb_mess->mess + sizeof(group_id) + sizeof(int));
+
+  /* fill in body of msg: vs_membs */
+  if (not_network)
+    memcpy(curr_vs, group->curr_change->delta_member, MAX_GROUP_NAME);    
+  else {
+    char (*old_grps)[MAX_GROUP_NAME]     = last_fl_view->membs_names;
+    char (*old_grps_end)[MAX_GROUP_NAME] = old_grps + last_fl_view->orig_num_membs;
+    stdit hit;
+    
+    /* Step through the last fl_view's membs_names checking to see if each name is still */
+    /* in the membs (vs) set. This ensures the vs set is deterministically ordered. */
+    for (; old_grps != old_grps_end; ++old_grps)
+      if (!stdhash_is_end(&last_fl_view->curr_membs, stdhash_find(&last_fl_view->curr_membs, &hit, &old_grps)))
+	memcpy(curr_vs++, old_grps, MAX_GROUP_NAME);
+  }
+  deliver(conn, m, memb_mess, 1);
+  DEBUG(std_stkfprintf(stderr, -1, "install_new_view: mbox(%d, %p), group('%s', %p), %s\n",
+		       conn->mbox, conn, group->group, group, state_str(group->vstate)));
+}
+
+static void update_fl_view(fl_group *group) {
+  stdit hit;
+  stdit lit;
+
+  DEBUG(std_stkfprintf(stderr, 1, "update_fl_view: group('%s', %p), %s, old gid %d %d %d\n",
+		       group->group, group, state_str(group->vstate), group->fl_view->gid.id[0],
+		       group->fl_view->gid.id[1], group->fl_view->gid.id[2]));
+  free_view(group->fl_view);                                         /* delete old fl_view */
+  group->fl_view = group->curr_change->memb_info;              /* steal curr_change's view */
+  group->curr_change->memb_info = 0;                  /* don't let fl_view be free'd below */
+
+  stddll_begin(&group->memb_queue, &lit);
+  assert(!stddll_is_end(&group->memb_queue, &lit) && 
+	 group->curr_change == *(sp_memb_change**) stddll_it_val(&lit));
+  stddll_erase(&group->memb_queue, &lit);                        /* remove curr_change from head of memb_queue */
+  assert(stddll_is_end(&group->memb_queue, &lit) || 
+	 group->curr_change != *(sp_memb_change**) stddll_it_val(&lit));
+
+  stdhash_find(&group->pmemb_hash, &hit, &group->fl_view->gid);
+  assert(!stdhash_is_end(&group->pmemb_hash, &hit) && 
+	 group->curr_change == *(sp_memb_change**) stdhash_it_val(&hit));
+  stdhash_erase(&group->pmemb_hash, &hit);                            /* remove curr_change from pmemb_hash */
+  assert(stdhash_is_end(&group->pmemb_hash, stdhash_find(&group->pmemb_hash, &hit, &group->fl_view->gid)));
+
+  free_sp_memb_change(group->curr_change);                     /* reclaim non-stolen parts */
+  group->curr_change = 0;
+  DEBUG(std_stkfprintf(stderr, -1, "update_fl_view: group('%s', %p), %s, new gid %d %d %d\n",
+		       group->group, group, state_str(group->vstate), group->fl_view->gid.id[0],
+		       group->fl_view->gid.id[1], group->fl_view->gid.id[2]));
+}
+
+static char*
+determine_leavers(stdhash *leavers, view *last_sp_view, gc_recv_mess *m) {
+  char delta[MAX_GROUP_NAME], *delta_ptr = delta, *ret = 0;
+  int num_vs, i, *num_vs_ptr = &num_vs;
+  scatter *scat;
+  scatp pos;
+  long err;
+
+  DEBUG(std_stkfprintf(stderr, 1, "determine_leavers: old gid %d %d %d -> new gid %d %d %d\n", 
+		       last_sp_view->gid.id[0], last_sp_view->gid.id[1], 
+		       last_sp_view->gid.id[2], m->dst_gid.id[0], m->dst_gid.id[1],
+		       m->dst_gid.id[2]));
+
+  get_scat_info(m, &i, &scat);                                   /* i is a dummy variable here */
+  if (!Is_caused_network_mess(*m->serv_type)) {               /* join, leave, or disconnection */
+    DEBUG(std_stkfprintf(stderr, 0, "Not caused by network! "));
+    stdhash_construct(leavers, sizeof(char*), 0, 
+		      group_name_ptr_cmp, group_name_ptr_hashcode, 0);
+
+    err = scatp_set(&pos, scat, sizeof(group_id) + sizeof(int), SEEK_SET);
+    assert(err == 0);
+    err = scatp_cpy1(delta, &pos, MAX_GROUP_NAME);      /* read out joiner/leaver/disconnector */
+    assert(err == MAX_GROUP_NAME);
+    delta[MAX_GROUP_NAME - 1] = 0;                                  /* ensure null termination */
+
+    if ((ret = (char*) malloc(MAX_GROUP_NAME)) == 0)
+      stderr_abort("(%s, %d): malloc(%d)\n", __FILE__, __LINE__, MAX_GROUP_NAME);
+    strncpy(ret, delta, MAX_GROUP_NAME);                    /* strndup delta member for return */
+
+    DEBUG(std_stkfprintf(stderr, 0, "delta memb: '%s'\n", delta));
+    if (!Is_caused_join_mess(*m->serv_type)) {                   /* insert leaver into leavers */
+      DEBUG(std_stkfprintf(stderr, 0, "Wasn't a JOIN: adding delta to leavers\n"));
+      stdhash_insert(leavers, 0, &ret, 0);
+    }
+  } else {
+    DEBUG(std_stkfprintf(stderr, 0, "Caused by network! "));
+    stdhash_copy_construct(leavers, &last_sp_view->orig_membs);     /* copy last SP membership */
+
+    err = scatp_set(&pos, scat, sizeof(group_id), SEEK_SET);
+    assert(err == 0);
+    err = scatp_adv_cpy1((char**) &num_vs_ptr, &pos, sizeof(int), 0, 1);    /* read out num_vs */
+    assert(err == sizeof(int));
+
+    DEBUG(std_stkfprintf(stderr, 0, "Num_vs %d\n", num_vs));
+    for (i = 0; i < num_vs; ++i) {        /* remove members that came with me: leaves who left */
+      err = scatp_adv_cpy1(&delta_ptr, &pos, MAX_GROUP_NAME, 0, 1);
+      assert(err == MAX_GROUP_NAME);
+      delta[MAX_GROUP_NAME - 1] = 0;                                /* ensure null termination */
+      err = stdhash_size(leavers);
+      stdhash_erase_key(leavers, &delta_ptr);
+      assert(err == stdhash_size(leavers) + 1);
+    }
+  }
+  DEBUG(std_stkfprintf(stderr, -1, "determine_leavers: delta('%s', %p), leavers size is %lu\n",
+		       ret != 0 ? ret : "", ret, stdhash_size(leavers)));
+  return ret;                                               /* strndup'ed delta member or null */
+}
+
+static void age_and_invalidate_pmembs(fl_group *group) {
+  sp_memb_change *spc;
+  stdit hit;
+
+  DEBUG(std_stkfprintf(stderr, 1, "age_and_invalidate_pmembs: group('%s', %p), %s\n",
+		       group->group, group, state_str(group->vstate)));
+
+  stdhash_begin(&group->pmemb_hash, &hit); 
+  for (; !stdhash_is_end(&group->pmemb_hash, &hit); stdhash_it_next(&hit)) {
+    spc = *(sp_memb_change**) stdhash_it_val(&hit);
+    if (!spc->memb_mess_recvd && ++spc->memb_change_age > FLOK_AGE_LIMIT) {
+      DEBUG(std_stkfprintf(stderr, 0, "destroy too old spc %p, %d %d %d\n", spc,
+			   spc->memb_info->gid.id[0], spc->memb_info->gid.id[1],
+			   spc->memb_info->gid.id[2]));
+      stdhash_erase(&group->pmemb_hash, &hit);
+      free_sp_memb_change(spc);
+      break;                             /* only one memb can become too old per SP memb event */
+    }
+  }
+  DEBUG(std_stkfprintf(stderr, -1, "age_and_invalidate_pmembs: group('%s', %p), %s\n",
+		       group->group, group, state_str(group->vstate)));
+}
+
+static sp_memb_change*
+collapse_memberships(fl_group *group, stdhash *leavers, sp_memb_change *c) {
+  stdit leavers_it, spc_it, flok_it, pmemb_it;
+  int invalidating = 0;
+  sp_memb_change *spc;
+  stdit lit;
+  char *leaver;
+
+  DEBUG(std_stkfprintf(stderr, 1, "collapse_memberships: group('%s', %p), %s, num_leavers %lu\n",
+		       group->group, group, state_str(group->vstate), stdhash_size(leavers)));
+  if (!stddll_empty(&group->memb_queue) && !stdhash_empty(leavers)) {
+    for (stddll_begin(&group->memb_queue, &lit); !stddll_is_end(&group->memb_queue, &lit); ) { /* loop SP membs */
+      spc = *(sp_memb_change**) stddll_it_val(&lit);
+      
+      stdhash_begin(leavers, &leavers_it);                                /* loop over leavers */
+      for (; !stdhash_is_end(leavers, &leavers_it); stdhash_it_next(&leavers_it)) {  
+	leaver = *(char**) stdhash_it_key(&leavers_it);
+
+	/* see if a leaver is a member of this SP memb change */
+	if (!stdhash_is_end(&spc->memb_info->curr_membs, stdhash_find(&spc->memb_info->curr_membs, &spc_it, &leaver))) {
+	  DEBUG(std_stkfprintf(stderr, 0, "leaver '%s' is in a pending membership!\n", leaver));
+	  /* if leaver hasn't sent his flok yet, then invalidate this sp_memb_change: NO & !!! */
+	  if (stdhash_is_end(&spc->flok_senders, stdhash_find(&spc->flok_senders, &flok_it, leaver))) {
+	    DEBUG(std_stkfprintf(stderr, 0, "leaver didn't send flok: collapsing membship!\n"));
+	    invalidating = 1;
+	    stdhash_find(&group->pmemb_hash, &pmemb_it, &spc->memb_info->gid);
+	    assert(!stdhash_is_end(&group->pmemb_hash, &pmemb_it));
+	    stdhash_erase(&group->pmemb_hash, &pmemb_it);       /* removes spc from pmemb hash */
+	    stddll_erase(&group->memb_queue, &lit);                /* removes spc from memb_queue and advances lit */
+	    free_sp_memb_change(spc);
+	    break;                             /* break out of inner loop: move on to next spc */
+	  } else             /* remove leaver from &spc->memb_info->curr_membs because he left */
+	    stdhash_erase(&spc->memb_info->curr_membs, &spc_it);
+	}
+      }
+      if (stdhash_is_end(leavers, &leavers_it)) { /* didn't break out of loop due to an invalidation */
+	if (invalidating) {                  /* did invalidate some spc's previous to this one */
+	  DEBUG(std_stkfprintf(stderr, 0, "collapsing invalidates here: CAUSED_BY_NETWORK\n"));
+	  invalidating = 0;
+	  spc->memb_info->memb_type = REG_MEMB_MESS | CAUSED_BY_NETWORK; /* collapse them here */
+	}
+	stddll_it_next(&lit);                            /* advance lit to next sp_memb_change */
+      }
+    }
+    if (invalidating) {        /* invalidated last few on the memb queue: collapse them into c */
+      DEBUG(std_stkfprintf(stderr, 0, "collapsing invalidates into current spc\n"));
+      c->memb_info->memb_type = REG_MEMB_MESS | CAUSED_BY_NETWORK;
+    }
+  }
+  spc = (stddll_empty(&group->memb_queue) ? c : 
+	 *(sp_memb_change**) stddll_it_val(stddll_begin(&group->memb_queue, &lit)));
+
+  DEBUG(std_stkfprintf(stderr, -1, "collapse_memberships: curr change %p, group('%s', %p), %s\n",
+		       spc, group->group, group, state_str(group->vstate)));
+  return spc;
+}
+
+static void get_groups_info(const gc_recv_mess *m, int *num_groups,
+				       char (**groups)[MAX_GROUP_NAME]) {
+  if (m->num_new_grps == 0) {
+    *num_groups = *m->num_groups;
+    *groups = m->groups;
+  } else {
+    *num_groups = m->num_new_grps;
+    *groups = m->new_grps;
+  }
+}
+
+static void get_scat_info(const gc_recv_mess *m, int *mess_len,
+				     scatter **scat_mess) {
+  if (m->new_msg.num_elements == 0) {
+    *mess_len = m->ret;
+    *scat_mess = (scatter*) m->scat_mess;
+  } else {
+    *mess_len = m->new_msg.elements[0].len;
+    *scat_mess = (scatter*) &m->new_msg;
+  }
+}
+
+/* is a group name a private group name or not? */
+static int is_private_group(const char group[MAX_GROUP_NAME]) {
+  const char *end = group + MAX_GROUP_NAME;
+
+  while (group != end && *group != 0 && *group != '#')
+    ++group;
+  
+  return group != end && *group != 0;
+}
+
+/* check if a msg wrt a particular group is vulnerable or not */
+static int is_vulnerable_mess(const fl_group *group, service serv_type) {
+  /* need to include AUTHORIZE state because of coming here directly from the VERIFY state */
+  return ((group->vstate == AUTHORIZE || group->vstate == VERIFY) &&
+	  (serv_type & (UNRELIABLE_MESS | RELIABLE_MESS | FIFO_MESS)) != 0);
+}
+
+static const char *state_str(fl_view_state state) {
+  switch (state) {
+  case STEADY:    return "STEADY";
+  case AUTHORIZE: return "AUTHORIZE";
+  case AGREE:     return "AGREE";
+  case VERIFY:    return "VERIFY";
+  default:        return "UNKOWN STATE!!!\n";
+  }
+}
+
+static float FL_SP_version(void) {
+#ifdef SPREAD_VERSION
+  int major, minor, patch, divBy;
+
+  SP_version(&major, &minor, &patch);
+
+  if (minor < 100)
+    divBy = 100;
+  else
+    divBy = 1000;
+
+  return major + (float) minor / divBy;
+#else
+  return SP_version();
+#endif
+}
+
+/* hashing fcns for group names */
+static stdbool group_name_cmp(const void *str1, const void *str2) {
+  return strncmp((const char*) str1, (const char*) str2, MAX_GROUP_NAME);
+}
+
+static size_t group_name_hashcode(const void *str) {
+  const stduint8 *key = (const stduint8*) str, *end = key + MAX_GROUP_NAME;
+  size_t ret = 0;
+  
+  while (key != end && *key != 0) 
+    ret = ret * 33 + *key++;
+  
+  return ret;
+}
+
+static stdbool group_name_ptr_cmp(const void *strptr1, const void *strptr2) {
+  return strncmp(*(const char**) strptr1, *(const char**) strptr2, MAX_GROUP_NAME);
+}
+
+static size_t group_name_ptr_hashcode(const void *strptr) {
+  return group_name_hashcode(*(const void**) strptr);
+}
+
+static stdbool group_id_cmp(const void *gid1, const void *gid2) {
+  return memcmp(gid1, gid2, sizeof(group_id));
+}
+
+static size_t group_id_hashcode(const void *gid) {
+  group_id *gid2 = (group_id*) gid;
+
+  return (gid2->id[0] ^ gid2->id[1] ^ gid2->id[2]);
+}
+

Added: trunk/flush/fl_p.h
===================================================================
--- trunk/flush/fl_p.h	2005-07-28 14:50:56 UTC (rev 252)
+++ trunk/flush/fl_p.h	2005-07-31 10:51:40 UTC (rev 253)
@@ -0,0 +1,336 @@
+/*
+ * The contents of this file are subject to the FLUSH SPREAD Non-Commercial 
+ * License, Version 1.0 (the ``License''); you may not use this file except 
+ * in compliance with the License. You may obtain a copy of the License at:
+ *
+ * http://www.cnds.jhu.edu/research/group/flush_spread/FLUSH_LICENSE
+ *
+ * or in the file ``FLUSH_LICENSE'' found in the root of this distribution.
+ *
+ * Software distributed under the License is distributed on an AS IS basis, 
+ * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License 
+ * for the specific language governing rights and limitations under the 
+ * License.
+ *
+ * The Original Software is:
+ *    The Flush Spread Library
+ *     
+ * The Initial Developers of the Original Software are:
+ *    Yair Amir, John Schultz and Jonathan Stanton
+ *
+ *    All Rights Reserved.
+ *
+ */
+
+#ifndef flush_p_h_2000_04_24_14_16_31_jschultz_at_cnds_jhu_edu
+#define flush_p_h_2000_04_24_14_16_31_jschultz_at_cnds_jhu_edu
+
+/* In this library I assume that the stdutil library I am linking with
+   was compiled with the -D STD_USE_EXCEPTIONS flag. This allows me to
+   assume that all reasonable calls to stdutil fcns won't fail, and if
+   they do (due to a system problem, such as malloc failing) they abort
+   with an appropriate error message, line number, etc.
+*/
+#include <fl.h>
+#include <scatp.h>
+#include <stdutil/stddefines.h>
+#include <stdutil/stderror.h>
+#include <stdutil/stdthread.h>
+#include <stdutil/stddll.h>
+#include <stdutil/stdhash.h>
+
+#define FL_MAJOR_VERSION 1
+#define FL_MINOR_VERSION 0
+#define FL_PATCH_VERSION 3
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/* The type fl_view_state specifies the possible states a connection
+   can be in, in reference to a particular group's membership algorithm.
+
+   STEADY    - normal operation, member of the group
+   AUTHORIZE - a SP lvl memb event has occured, user needs to flush this group
+   AGREE     - user has flushed this group, collecting floks for next fl view
+   VERIFY    - new flush view installed, protect against non-causal msgs
+*/
+typedef enum { STEADY, AUTHORIZE, AGREE, VERIFY } fl_view_state;
+
+/* The type member_join_state specifies the possible states a connection 
+   can be in, in reference to being a member of a particular group. Only
+   JOINED connections are allowed to leave and send to a group.
+
+   JOINING - this connection is in the process of joining this group
+   JOINED  - this connection has been installed in a fl view of this group 
+   LEAVING - this connection is leaving this group 
+*/
+typedef enum { JOINING, JOINED, LEAVING } member_join_state;
+
+/* A gc_buff_mess contains all of the information necessary to replicate a message. */
+/* This is the struct I use when I have to buffer messages. */
+typedef struct {
+  mailbox mbox;
+  service serv_type;
+  char    sender[MAX_GROUP_NAME];
+  int     num_groups;
+  char    (*groups)[MAX_GROUP_NAME];
+  int16   mess_type;
+  int     endian_mismatch;
+  int     mess_len;
+  char    *mess;
+  int     total_size;         /* used for reporting number of bytes buffered: set by deliver() */
+} gc_buff_mess;
+
+/* A gc_recv_mess contains references to user's parameters from a call to receive.        */
+/* I use this struct to pass data around in the internal state machine code.              */
+/* The struct member "delivered" indicates whether or not the user's parameters have been */
+/* filled with legal return values and is currently returnable to the user or not.        */
+typedef struct {
+  mailbox  mbox;
+  service  orig_serv_type;                  /* user's original service type request to receive */
+  service  *serv_type;
+  char     *sender;
+  int      max_groups;
+  int      *num_groups;
+  char     (*groups)[MAX_GROUP_NAME];
+  int16    *mess_type;
+  int      *endian_mismatch;
+  scatter  *scat_mess;
+  int      ret;                                              /* return value from receive call */
+
+  int      delivered;                  /* does this recv_mess already contain a delivered msg? */
+  int      vulnerable;                                              /* is this msg vulnerable? */
+  group_id dst_gid;                                /* the view id this message is addressed to */
+
+  int      num_new_grps;                    /* potentially alloc'ed buffers for DROP_RECV msgs */
+  char     (*new_grps)[MAX_GROUP_NAME];
+  scatter  new_msg;
+} gc_recv_mess;
+
+/* This struct contains information that specifies a group membership view. */
+typedef struct {
+  group_id gid;
+  service  memb_type;
+  int      in_trans_memb;               /* has a transitional msg been delivered in this view? */
+  int      orig_num_membs;
+  char     (*membs_names)[MAX_GROUP_NAME];
+  stdhash  orig_membs, curr_membs;                      /* <char*, nil>, ptrs into membs_names */
+  int16    my_index;                                /* index of this connection in membs_names */
+} view;
+
+/* This struct contains information on a spread level membership
+   change for a particular group that has not yet been fully handled
+   by the flush layer. This structure is used by the fl memb protocol
+   while it attempts to install a view.  These structures are built in
+   two different scenarios: (1) a FLUSH_OK or (2) a spread membership
+   message is received for a previously unseen/unknown view id.
+
+   There are 2 states that this structure can be in. These two states
+   correspond to whether or not a spread membership message for the
+   particular view id has been received yet or not. Which state the
+   structure is in is indicated by the struct member memb_mess_recvd.
+
+   The struct member memb_change_age indicates how many spread
+   membership messages have been received in this group since this
+   structure was built. If this value exceeds the value FLOK_AGE_LIMIT
+   and !memb_mess_recvd, then I take this as an indication that the
+   spread membership message that this structure is waiting for was
+   already delivered in a view of which I wasn't a member. In that
+   case, I will never receive that membership message and I can
+   "safely" discard this struct, as that membership change was already
+   handled by other members (this assumption is not 100% correct as
+   FIFO messages can jump before an unbounded number of memberships;
+   however, I feel that the probability of a FIFO message jumping
+   before FLOK_AGE_LIMIT memberships is vanishingly small).
+
+   If !memb_mess_recvd then the only fields that contain meaningful data
+   are memb_change_age, flok_senders, and memb_info->gid.
+
+   If memb_mess_recvd then delta_member contains either (1) a
+   malloc'ed copy of the delta member (if the membership change was
+   caused by join/leave/disconnect) or (2) null. All of the info in
+   memb_info is appropriately filled in when in this state.
+
+   The struct member flok_senders contains by value (ie - alloc'ed by
+   flok_senders) members' names who have flok'ed to this view_id.  
+*/
+typedef struct {
+  view     *memb_info;
+  int      memb_mess_recvd;
+  int      memb_change_age;
+  char     *delta_member;
+  stdhash  flok_senders;                                         /* <char[MAX_GROUP_NAME], nil> */
+  group_id gid;             /* temporary: erase me */
+} sp_memb_change;
+
+/* This struct contains all of the group level context for a connection in 
+   reference to a particular group. 
+
+   group_name  - the name of the group with which this information is associated
+   mstate      - current join state this member is in
+   vstate      - current fl memb protocol state this member is in
+   curr_change - points to current sp_memb_change fl is trying to resolve (head of memb_queue)
+   sp_view     - my most recently installed spread membership view (initially just me)
+   fl_view     - my most recently installed flush membership view (initially just me)
+   flush_recvs - a count of how many flush_recvs have been recvd for fl_view
+   mess_queue  - a queue of gc_buff_mess*s to be delivered later, in order
+   memb_queue  - a queue of sp_memb_change*s to be handled in order
+   pmemb_hash  - sp_memb_change*s that are being handled currently
+
+   sp_view either points at fl_view or at the most recent pending SP
+   memb event's view in memb_queue (see handle_recv_reg_memb_mess)
+
+   fl_view steals curr_change's view when curr_change is installed (see update_fl_view)
+*/
+typedef struct {
+  char group[MAX_GROUP_NAME];
+
+  member_join_state mstate;
+  fl_view_state     vstate;
+
+  sp_memb_change *curr_change;
+
+  view *sp_view;
+  view *fl_view;
+  int flush_recvs;
+
+  stddll  mess_queue;                                                /* <gc_buff_mess*> */
+  stddll  memb_queue;                                              /* <sp_memb_change*> */
+  stdhash pmemb_hash;                                    /* <group_id, sp_memb_change*> */
+} fl_group;
+
+/* This struct contains all of the connection level context for a connection */
+typedef struct {
+  stdmutex reserve_lock;  /* protects reservations, disconnecting; destroy_cond's mutex */
+  size_t   reservations;                    /* count of reservations on this connection */
+  int      disconnecting;                     /* is this connection being disconnected? */
+  stdcond  destroy_cond;   /* disconnector thread waits on this until reservations == 0 */
+
+  stdmutex recv_lock;             /* used to serialize calls to FL_recv on a connection */
+  stdmutex conn_lock;            /* lock for access to the connection's data, see below */
+
+  /* acquiring conn_lock allows a thread to examine everything below here */
+  mailbox mbox;           
+  int     priority;
+  int     group_memb;
+  char    daemon[MAX_GROUP_NAME];
+  char    user[MAX_GROUP_NAME];
+  char    private[MAX_GROUP_NAME];
+
+  /* acquiring conn_lock allows a thread to examine and modify everything below here */
+  stdhash groups;              /* information on groups involved in: <char*, fl_group*> */
+  stddll  mess_queue;           /* messages that the user can read out: <gc_buff_mess*> */
+  int     bytes_queued;     /* number of bytes available in mess_queue, used by FL_poll */
+} fl_conn;
+
+/************************* Private Variables, Functions and Macros *****************************/
+
+/* leave in or take out debugging printf's and checks? */
+#ifdef NDEBUG
+# define DEBUG(statement)
+#else
+# define DEBUG(statement) statement
+#endif
+
+/* age limit for a sp_memb_change with no sp memb mess */
+#define FLOK_AGE_LIMIT 200   
+
+#define IS_ILLEGAL_SEND_STYPE(serv_type) \
+(((serv_type) & (FLUSH_REQ_MESS | MEMBERSHIP_MESS | ENDIAN_RESERVED | RESERVED)) != 0)
+
+/* mess types reserved by flush layer indicating special flush messages */
+/* MAKE SURE THAT FL_MIN_LEGAL_MESS_TYPE (in fl.h) IS ONE MORE THAN THE HIGHEST RESERVED MESS TYPE!!! */
+#define FLUSH_OK_MESS                    ((int16) -32768)
+#define FLUSH_RECV_MESS                  ((int16) -32767)
+#define VULNERABLE_MESS                  ((int16) -32766)
+#define IS_ILLEGAL_SEND_MTYPE(mess_type) ((mess_type) < FL_MIN_LEGAL_MESS_TYPE)
+
+/* structure routines */
+static fl_group *create_fl_group(const char *conn_name, const char *group_name);
+static void free_fl_group(fl_group *group);
+
+static void free_gc_buff_mess(gc_buff_mess *msg);
+
+static sp_memb_change *create_sp_memb_change(group_id gid);
+static void free_sp_memb_change(sp_memb_change *change);
+
+static view *create_view(group_id gid);
+static void free_view(view *v);
+static void fill_view(view *v, service memb_type, int num_membs, 
+			     char (*membs)[MAX_GROUP_NAME], int16 index);
+
+/* interface for reserving/locking connections */
+static fl_conn *lock_conn(mailbox mbox);             /* reserve and lock connection */
+static void    unlock_conn(fl_conn *conn);
+static fl_conn *make_reservation(mailbox mbox);   /* declare interest in connection */
+static void    cancel_reservation(fl_conn *conn);
+static int     acquire_conn_lock(fl_conn *conn);       /* get rights to read/modify */
+static void    release_conn_lock(fl_conn *release);
+static int     acquire_recv_lock(fl_conn *conn);     /* get rights to enter FL_recv */
+static void    release_recv_lock(fl_conn *release);
+
+/* interface for groups for a connection */
+static fl_group *add_group(fl_conn *conn, const char *group);
+static void      remove_group(fl_conn *conn, fl_group *group);
+static fl_group *get_group(fl_conn *conn, const char *grp);
+
+/* functions for trying to deliver/return msgs to the user */
+static gc_buff_mess *deliver(fl_conn *c, gc_recv_mess *um, gc_buff_mess *bm, int al);
+static gc_buff_mess *deliver_trans_sig(fl_conn *conn, gc_recv_mess *um, char *group);
+static gc_buff_mess *deliver_flush_req(fl_conn *conn, gc_recv_mess *um, char *group);
+
+/* converting back and forth between user's variables and buffered messages (dst, src) */
+static int buffm_to_userm(gc_recv_mess *um, const gc_buff_mess *bm);
+static void userm_to_buffm(gc_buff_mess *bm, const gc_recv_mess *um);
+
+/* complex "low level" internal fcns that deal with SP layer */
+static int FL_int_flush(fl_conn *conn, fl_group *group);
+static int FL_int_scat_multicast(mailbox m, service s, const char *g,
+					int nr, char r[][MAX_GROUP_NAME],
+					int16 mess_type, const scatter *scat);
+static int FL_int_receive(gc_recv_mess *m);
+
+/* state machine fcns: handler functions */
+static int  recv_and_handle(fl_conn *c, gc_recv_mess *m);
+static void handle_recv_flush_ok(fl_conn *c, fl_group *g, gc_recv_mess *m);
+static void handle_recv_flush_recv(fl_conn *c, fl_group *g, gc_recv_mess *m);
+static void handle_recv_reg_mess(fl_conn *c, fl_group *g, gc_recv_mess *m);
+static void handle_recv_reg_memb_mess(fl_conn *c, fl_group *g, gc_recv_mess *m);
+static void handle_next_memb_change(fl_conn *c, fl_group *g, gc_recv_mess *m);
+static void handle_recv_trans_memb_mess(fl_conn *c, fl_group *g, gc_recv_mess *m);
+static void handle_recv_self_leave_memb_mess(fl_conn *c, fl_group *g, gc_recv_mess *m);
+
+/* helper functions for handler functions */ 
+static void  install_new_view(fl_conn *c, const fl_group *g, gc_recv_mess *m);
+static void  update_fl_view(fl_group *group);
+static char *determine_leavers(stdhash *leavers, view *last_sp_view, gc_recv_mess *m);
+static void  age_and_invalidate_pmembs(fl_group *group);
+static sp_memb_change *collapse_memberships(fl_group *group, stdhash *leavers, 
+						   sp_memb_change *c);
+/* miscellaneous functions */
+/* where is full data for a msg (ie - groups or new_grps?) due to DROP_RECV considerations */
+static void get_groups_info(const gc_recv_mess *m, int *num_groups, 
+				   char (**groups)[MAX_GROUP_NAME]);
+static void get_scat_info(const gc_recv_mess *m, int *mess_len, scatter **scat_mess);
+
+static int is_private_group(const char *group);
+static int is_vulnerable_mess(const fl_group *group, service serv_type);
+
+static const char *state_str(fl_view_state state);
+
+static float FL_SP_version(void);
+
+/* group name hash fcns: used by stdhashs where the keys are group_names */
+static stdbool group_name_cmp(const void *str1, const void *str2);
+static size_t  group_name_hashcode(const void *str);
+static stdbool group_name_ptr_cmp(const void *str_ptr1, const void *str_ptr2);
+static size_t  group_name_ptr_hashcode(const void *str_ptr);
+static stdbool group_id_cmp(const void *gid1, const void *gid2);
+static size_t  group_id_hashcode(const void *gid);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif

Added: trunk/flush/fl_time_memb.c
===================================================================
--- trunk/flush/fl_time_memb.c	2005-07-28 14:50:56 UTC (rev 252)
+++ trunk/flush/fl_time_memb.c	2005-07-31 10:51:40 UTC (rev 253)
@@ -0,0 +1,261 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <math.h>
+#include <limits.h>
+#include <float.h>
+
+#include <sys/time.h>
+#include <sys/resource.h>
+#include <unistd.h>
+
+#include "fl.h"
+#include "stats.h"
+
+#define MY_MAX_NUM_GROUPS 1000
+#define MY_MAX_MESS_SIZE 102400
+
+enum { LOAD_MEMBER = 0, DELTA = 1, DIE_MESS = 2048 };
+
+mailbox mbox;
+service service_type;
+char    sender[MAX_GROUP_NAME];
+int     num_groups;
+char    groups[MY_MAX_NUM_GROUPS][MAX_GROUP_NAME];
+int16   mess_type;
+int     endian_mismatch;
+int     mess_len;
+char    mess[MY_MAX_MESS_SIZE];
+int     more_messes;
+
+char    daemon_name[MAX_GROUP_NAME] = "4803 at localhost";
+char    user_name[MAX_GROUP_NAME]   = "1";
+char    priv_name[MAX_GROUP_NAME];
+char    group_name[MAX_GROUP_NAME]  = "test";
+
+char *exe = 0;             /* executable's name */
+int user_type = -1;        /* user type of application */
+int num_joins_leaves = 0;  /* i: tmp var, num_joins_leaves: # of memberships to cause */
+int about_to_die = 0;      /* boolean: am I dieing after next membership? */
+int num_members;
+
+int should_sleep = 1;      /* should there be sleeps between memberships? Ususally yes. */
+int pretty_print = 1;      /* should the output be verbose and labeled for human consumption? */
+
+/* the results of the scenarios timings */
+stats_results sp_join_stats, sp_leave_stats, sp_cmbo_stats;
+stats_results fl_join_stats, fl_leave_stats, fl_cmbo_stats;
+
+int i, err, num_joins;
+
+static int printUsage(FILE *outstream) {
+  return fprintf(outstream, 
+		 "Usage: %s\r\n"
+		 "\t[-S <group size>]                  : # members in group (for stats; also login name)\r\n"
+		 "\t[-s <address>]                     : spread daemon name - either port or port at machine\r\n"
+		 "\t[-g <group name>]                  : group name to join\r\n"
+		 "\t[-t <user type> <# joins/leaves>]  : type of user "
+		 "(LOAD_MEMBER = %d, DELTA = %d), # of join/leave events\r\n"
+		 "\t[-f]                               : don't sleep between memberships (default is to sleep)\r\n"
+		 "\t[-r]                               : print raw stats w/ no pretty headings\r\n", 
+		 exe, LOAD_MEMBER, DELTA);
+}
+
+static void usage(int argc, char **argv) {
+  for (++argv, --argc; argc > 0; ++argv, --argc) {
+    if (!strcmp(*argv, "-S") && --argc) {
+      strncpy(user_name, *++argv, MAX_GROUP_NAME);
+
+    } else if (!strcmp(*argv, "-s") && --argc) {
+      strncpy(daemon_name, *++argv, MAX_GROUP_NAME);
+
+    } else if (!strcmp(*argv, "-g") && --argc) {
+      strncpy(group_name, *++argv, MAX_GROUP_NAME);
+
+    } else if (!strcmp(*argv, "-t") && (argc -= 2) > 0) {
+      user_type = atoi(*++argv);
+      num_joins_leaves = atoi(*++argv); 
+
+    } else if (!strcmp(*argv, "-f")) {
+      should_sleep = 0;
+
+    } else if (!strcmp(*argv, "-r")) {
+      pretty_print = 0;
+
+    } else {
+      fprintf(stderr, "Unknown cmd line param: %s\r\n", *argv);
+      exit(printUsage(stderr));
+    }
+  }
+  num_members = atoi(user_name);  
+}
+
+int main(int argc, char **argv) {
+  double t;
+
+  exe = *argv;
+  usage(argc, argv);
+
+  if ((err = FL_connect(daemon_name, user_name, 0, &mbox, priv_name)) != ACCEPT_SESSION) {
+    fprintf(stderr, "FL_connect failure: ");
+    FL_error(err);
+    exit(1);
+  }  
+  if (user_type == DELTA) {
+    double *fl_join_times  = (double*) malloc(sizeof(double) * num_joins_leaves);
+    double *fl_leave_times = (double*) malloc(sizeof(double) * num_joins_leaves);
+    double *fl_cmbo_times  = (double*) malloc(sizeof(double) * num_joins_leaves);
+
+    if (!fl_join_times || !fl_leave_times || !fl_cmbo_times) {
+      exit(fprintf(stderr, "Couldn't mallocate tracking arrays!\r\n"));
+    }
+
+    for (i = 0; i < num_joins_leaves; ++i) {
+      t = get_time_timeofday();
+
+      if ((err = FL_join(mbox, group_name)) < 0) {
+	fprintf(stderr, "FL_join failure: ");
+	FL_error(err);
+	exit(1);
+      }	
+      do { 
+	if ((mess_len = FL_receive(mbox, &service_type, sender, MY_MAX_NUM_GROUPS, 
+				   &num_groups, groups, &mess_type, &endian_mismatch, 
+				   MY_MAX_MESS_SIZE, mess, &more_messes)) < 0) {
+	  fprintf(stderr, "FL_receive failure: ");
+	  FL_error(mess_len);
+	  exit(1);
+	}
+      } while (!Is_reg_memb_mess(service_type)); 
+
+      fl_join_times[i] = get_time_timeofday() - t;
+
+      if (should_sleep) {
+	/* sleep for 4 times how long the join membership took */
+	/* allows membership to stabilize */
+	usleep((unsigned long) (4 * fl_join_times[i] * 1000));
+      }
+      
+      /* send a kill message if we are done */
+      if (i == num_joins_leaves - 1) {
+	if ((mess_len = FL_multicast(mbox, SAFE_MESS, group_name, DIE_MESS, 0, 0)) < 0) {
+	  fprintf(stderr, "FL_multicast failure: ");
+	  FL_error(err);
+	  exit(1);
+	}
+      }      
+
+      t = get_time_timeofday();
+      
+      if ((err = FL_leave(mbox, group_name)) < 0) {
+	fprintf(stderr, "FL_leave failure: ");
+	FL_error(err);
+	exit(1); 
+      }	
+      do {
+	if ((mess_len = FL_receive(mbox, &service_type, sender, MY_MAX_NUM_GROUPS, 
+				   &num_groups, groups, &mess_type, &endian_mismatch, 
+				   MY_MAX_MESS_SIZE, mess, &more_messes)) < 0) {
+	  fprintf(stderr, "FL_receive failure: ");
+	  FL_error(mess_len);
+	  exit(1);
+	}
+      } while (!Is_self_leave(service_type));
+
+      fl_leave_times[i] = get_time_timeofday() - t;
+      fl_cmbo_times[i]  = fl_join_times[i] + fl_leave_times[i];
+
+      if (should_sleep) {
+	/* sleep for 4 times how long the join membership took */
+	/* allows membership to stabilize */
+	usleep((unsigned long) (4 * fl_join_times[i] * 1000));
+      }
+    }
+    /* compute timing statistics */
+    comp_stats(&fl_join_stats, fl_join_times, num_joins_leaves);
+    comp_stats(&fl_leave_stats, fl_leave_times, num_joins_leaves);
+    comp_stats(&fl_cmbo_stats, fl_cmbo_times, num_joins_leaves);
+
+    /* output timing statistics */
+    if (pretty_print) { 
+      printf("Flush Membership Timings (includes Spread): Group Size: %d, # Joins/Leaves: %d\r\n\r\n", 
+	     num_members, num_joins_leaves);
+
+      printf("\tFlush Join: ind. total (90%%): %.6fms, ind. total: %.6fms\r\n", 
+	     sp_join_stats.total90, sp_join_stats.total);
+
+      printf("\tFlush Leave: ind. total (90%%): %.6fms, ind. total: %.6fms\r\n", 
+	     sp_leave_stats.total90, sp_leave_stats.total);
+
+      printf("\tFlush Join/Leave: ind. total (90%%): %.6fms, ind. total: %.6fms\r\n", 
+	     sp_cmbo_stats.total90, sp_cmbo_stats.total);
+
+      printf("\r\n\t\tGroup Size\t# Trials\tQ1\tMEDIAN\tQ3"
+	     "\tMIN (5%%)\tMEAN90\tSTDDEV90\tMAX (95%%)"
+	     "\tMIN\tMEAN\tSTDDEV\tMAX\r\n");
+    }
+    printf("\tFL_Join:\t%d\t%d\t%.6f\t%.6f\t%.6f\t%.6f\t%.6f\t%.6f\t%.6f\t%.6f\t%.6f\t%.6f\t%.6f\r\n",
+	   num_members, num_joins_leaves, fl_join_stats.quart1, fl_join_stats.median,
+	   fl_join_stats.quart3, fl_join_stats.min5, fl_join_stats.mean90, 
+	   fl_join_stats.stddev90, fl_join_stats.max95, fl_join_stats.min, 
+	   fl_join_stats.mean, fl_join_stats.stddev, fl_join_stats.max);
+
+    printf("\tFL_Leave:\t%d\t%d\t%.6f\t%.6f\t%.6f\t%.6f\t%.6f\t%.6f\t%.6f\t%.6f\t%.6f\t%.6f\t%.6f\r\n",
+	   num_members, num_joins_leaves, fl_leave_stats.quart1, fl_leave_stats.median,
+	   fl_leave_stats.quart3, fl_leave_stats.min5, fl_leave_stats.mean90, 
+	   fl_leave_stats.stddev90, fl_leave_stats.max95, fl_leave_stats.min, 
+	   fl_leave_stats.mean, fl_leave_stats.stddev, fl_leave_stats.max);
+
+    printf("\tFL_Join + FL_Leave\t%d\t%d\t%.6f\t%.6f\t%.6f\t%.6f\t%.6f\t%.6f\t%.6f\t%.6f\t%.6f\t%.6f\t%.6f\r\n",
+	   num_members, num_joins_leaves, fl_cmbo_stats.quart1, fl_cmbo_stats.median,
+	   fl_cmbo_stats.quart3, fl_cmbo_stats.min5, fl_cmbo_stats.mean90, 
+	   fl_cmbo_stats.stddev90, fl_cmbo_stats.max95, fl_cmbo_stats.min, 
+	   fl_cmbo_stats.mean, fl_cmbo_stats.stddev, fl_cmbo_stats.max);
+
+  } else if (user_type == LOAD_MEMBER) { 
+    if ((err = FL_join(mbox, group_name)) < 0) {
+      fprintf(stderr, "FL_join failure: ");
+      FL_error(err);
+      exit(1);
+    }      
+    while (1) {
+      if ((mess_len = FL_receive(mbox, &service_type, sender,
+				 MY_MAX_NUM_GROUPS, &num_groups, groups,
+				 &mess_type, &endian_mismatch,
+				 MY_MAX_MESS_SIZE, mess, &more_messes)) < 0) { 
+	fprintf(stderr, "FL_receive failure: ");
+	FL_error(mess_len);
+	exit(1);
+      }
+      if (Is_flush_req_mess(service_type)) {
+	if ((err = FL_flush(mbox, group_name)) < 0) {
+	  fprintf(stderr, "FL_flush failure: ");
+	  FL_error(err);
+	  exit(1);
+	}
+      } else if (Is_reg_memb_mess(service_type)) {
+	if (Is_caused_leave_mess(service_type)) {
+	  if (about_to_die) {
+	    break;
+	  }
+	} else if (!Is_caused_join_mess(service_type)) {
+	  exit(fprintf(stderr, "Unexpected membership type: %d\n", service_type));
+	}
+      } else if (Is_safe_mess(service_type) && mess_type == DIE_MESS) {
+	about_to_die = 1;
+      }
+    }
+    printf("Success!\r\n");
+
+  } else { 
+    fprintf(stderr, "Unknown user type: %d\n", user_type); 
+    exit(printUsage(stderr));
+  }
+  
+  if ((err = FL_disconnect(mbox)) < 0) {
+    fprintf(stderr, "FL_disconnect failure: ");
+    FL_error(err);
+    exit(1);
+  }
+  return 0;
+}

Added: trunk/flush/scatp.c
===================================================================
--- trunk/flush/scatp.c	2005-07-28 14:50:56 UTC (rev 252)
+++ trunk/flush/scatp.c	2005-07-31 10:51:40 UTC (rev 253)
@@ -0,0 +1,440 @@
+/*
+ * The contents of this file are subject to the FLUSH SPREAD Non-Commercial 
+ * License, Version 1.0 (the ``License''); you may not use this file except 
+ * in compliance with the License. You may obtain a copy of the License at:
+ *
+ * http://www.cnds.jhu.edu/research/group/flush_spread/FLUSH_LICENSE
+ *
+ * or in the file ``FLUSH_LICENSE'' found in the root of this distribution.
+ *
+ * Software distributed under the License is distributed on an AS IS basis, 
+ * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License 
+ * for the specific language governing rights and limitations under the 
+ * License.
+ *
+ * The Original Software is:
+ *    The Flush Spread Library
+ *     
+ * The Initial Developers of the Original Software are:
+ *    Yair Amir, John Schultz and Jonathan Stanton
+ *
+ *    All Rights Reserved.
+ *
+ */
+
+#include <errno.h>
+#include <assert.h>
+#include <string.h>
+#include <scatp.h>
+
+long scat_capacity(const scatter *scat) 
+{
+  const scat_element *curr = scat->elements, *end = scat->elements + scat->num_elements;
+  long ret = 0;
+
+  if (scat->num_elements < 0 || scat->num_elements > MAX_CLIENT_SCATTER_ELEMENTS)
+    return ILLEGAL_MESSAGE;
+
+  for (; curr != end; ++curr) {
+    if (curr->len < 0)
+      return ILLEGAL_MESSAGE;
+    ret += curr->len;
+  }
+  return ret;
+}
+
+int scatp_begin(scatp *pos, const scatter *scat) 
+{
+  int i;
+
+  if (scat->num_elements < 0 ||  scat->num_elements > MAX_CLIENT_SCATTER_ELEMENTS)
+    return ILLEGAL_MESSAGE;
+
+  for (i = 0; i < scat->num_elements && scat->elements[i].len == 0; ++i);
+
+  if (i != scat->num_elements && scat->elements[i].len < 0)
+    return ILLEGAL_MESSAGE;
+
+  pos->scat     = (scatter*) scat;
+  pos->elem_ind = i;
+  pos->buff_ind = 0;
+  return 0;
+}
+
+int scatp_end(scatp *pos, const scatter *scat) 
+{
+  if (scat->num_elements < 0 || scat->num_elements > MAX_CLIENT_SCATTER_ELEMENTS)
+    return ILLEGAL_MESSAGE;
+
+  pos->scat     = (scatter*) scat;
+  pos->elem_ind = scat->num_elements;
+  pos->buff_ind = 0;
+  return 0;
+}
+
+int scatp_set(scatp *pos, const scatter *scat, long offset, int whence) 
+{
+  int err;
+
+  if (whence == SEEK_CUR) {
+    if ((err = scatp_begin(pos, scat)))
+      return err;
+  } else
+    pos->scat = (scatter*) scat;
+
+  return scatp_seek(pos, offset, whence);
+}
+
+int scatp_is_legal(const scatp *pos) 
+{
+  const scatter *scat = pos->scat;
+
+  return (scat->num_elements >= 0 && scat->num_elements <= MAX_CLIENT_SCATTER_ELEMENTS &&
+	  (scatp_is_end(pos) ||
+	   (pos->elem_ind >= 0 && pos->elem_ind < scat->num_elements &&
+	    pos->buff_ind >= 0 && pos->buff_ind < scat->elements[pos->elem_ind].len)));
+}
+
+int scatp_is_not_legal(const scatp *pos) 
+{
+  const scatter *scat = pos->scat;
+
+  return (scat->num_elements < 0 || scat->num_elements > MAX_CLIENT_SCATTER_ELEMENTS ||
+	  (!scatp_is_end(pos) && 
+	   (pos->elem_ind < 0 || pos->elem_ind >= scat->num_elements || 
+	    pos->buff_ind < 0 || pos->buff_ind >= scat->elements[pos->elem_ind].len)));
+}
+
+int scatp_is_end(const scatp *pos) 
+{
+  if (pos->scat->num_elements < 0 || pos->scat->num_elements > MAX_CLIENT_SCATTER_ELEMENTS)
+    return ILLEGAL_MESSAGE;
+
+  return pos->elem_ind == pos->scat->num_elements && pos->buff_ind == 0;
+}
+
+int scatp_equals(const scatp *pos1, const scatp *pos2) 
+{
+  if (scatp_is_not_legal(pos1) || scatp_is_not_legal(pos2))
+    return ILLEGAL_MESSAGE;
+
+  if (pos1->scat != pos2->scat)
+    return ILLEGAL_SERVICE;
+
+  return pos1->elem_ind == pos2->elem_ind && pos1->buff_ind == pos2->buff_ind;
+}
+
+long scatp_comp(const scatp *pos1, const scatp *pos2) 
+{
+  const scat_element *curr, *end;
+  const scatter *scat = pos1->scat;
+  long ret;
+
+  if (scatp_is_not_legal(pos1) || scatp_is_not_legal(pos2))
+    return ILLEGAL_MESSAGE;
+
+  if (pos1->scat != pos2->scat)
+    return ILLEGAL_SERVICE;
+
+  if (pos1->elem_ind == pos2->elem_ind)
+    return pos1->buff_ind - pos2->buff_ind;
+
+  if (pos1->elem_ind < pos2->elem_ind) {
+    ret  = pos1->buff_ind - scat->elements[pos1->elem_ind].len;
+    curr = scat->elements + pos1->elem_ind;
+    end  = scat->elements + pos2->elem_ind;
+    while (++curr != end) {
+      if (curr->len < 0)
+	return ILLEGAL_MESSAGE;
+      ret -= curr->len;
+    }
+  } else {
+    ret  = scat->elements[pos2->elem_ind].len - pos2->buff_ind;
+    curr = scat->elements + pos2->elem_ind;
+    end  = scat->elements + pos1->elem_ind;
+    while (++curr != end) {
+      if (curr->len < 0)
+	return ILLEGAL_MESSAGE;
+      ret += curr->len;
+    }
+  }
+  return ret;
+}
+
+/* This fcn moves a scat_pos forward num_bytes bytes in a scatter. On
+   success, this fcn returns num_bytes and pos is modified
+   appropriately. Otherwise, if the jump was so big that it would have
+   jumped off the end of the scatter, the number of bytes that _could_
+   have been successfully jumped is returned, and pos is unchanged. 
+*/
+
+long scatp_jforward(scatp *pos, long num_bytes) 
+{ 
+  long elem_ind, skip_bytes, tmp; 
+  const scatter *scat = pos->scat;
+
+  if (scatp_is_not_legal(pos))
+    return ILLEGAL_MESSAGE;
+
+  if (num_bytes < 0)
+    return ILLEGAL_SERVICE;
+
+  if (scatp_is_end(pos)) /* can't move forward from end */
+    return 0;
+
+  /* jump stays within current element buffer */
+  if ((tmp = scat->elements[pos->elem_ind].len - pos->buff_ind) > num_bytes) {
+    pos->buff_ind += num_bytes;
+    return num_bytes;
+  }
+  /* incorporate what was left in current element buffer into jump */
+  elem_ind   = pos->elem_ind + 1;
+  skip_bytes = num_bytes - tmp;   /* how many bytes left to skip */
+  
+  for (; elem_ind < scat->num_elements; ++elem_ind) {
+    if (scat->elements[elem_ind].len < 0)
+      return ILLEGAL_MESSAGE;
+
+    /* use < 0 because it jumps over any zero length buffers */
+    if ((skip_bytes -= scat->elements[elem_ind].len) < 0) {
+      skip_bytes += scat->elements[elem_ind].len; /* restore to positive */
+      break;
+    }
+  }
+  /* jump forward jumped past end of scatter */
+  if (elem_ind == scat->num_elements && skip_bytes != 0) 
+    return num_bytes - skip_bytes;
+
+  pos->elem_ind = elem_ind;
+  pos->buff_ind = skip_bytes;
+  return num_bytes;
+}
+
+/* same as scat_jforward, except it moves the position backwards in the scatter */
+
+long scatp_jbackward(scatp *pos, long num_bytes) 
+{
+  long elem_ind, e_ind, skip_bytes;
+  const scatter *scat = pos->scat;
+
+  if (scatp_is_not_legal(pos))
+    return ILLEGAL_MESSAGE;
+
+  if (num_bytes < 0)
+    return ILLEGAL_SERVICE;
+
+  /* jump stays within current element buffer */
+  if (pos->buff_ind >= num_bytes) {
+    pos->buff_ind -= num_bytes;
+    return num_bytes;
+  }
+  /* incorporate what was left in current element buffer into jump */
+  elem_ind   = pos->elem_ind;
+  skip_bytes = num_bytes - pos->buff_ind;
+  
+  for (e_ind = pos->elem_ind - 1; e_ind >= 0; --e_ind) {
+    if (scat->elements[e_ind].len < 0)
+      return ILLEGAL_MESSAGE;
+
+    /* again we want to ignore any zero length buffers */
+    if (scat->elements[e_ind].len > 0) {
+      elem_ind = e_ind; /* elem_ind must reference a non-empty element buffer */
+      if ((skip_bytes -= scat->elements[e_ind].len) <= 0)
+	break;
+    }
+  }
+  if (e_ind < 0)
+    return num_bytes - skip_bytes;
+
+  pos->elem_ind = elem_ind;
+  pos->buff_ind = -skip_bytes; /* restore to positive */
+  return num_bytes;
+}
+
+int scatp_seek(scatp *pos, long offset, int whence) 
+{
+  scatp set;
+  long err;
+
+  switch (whence) {
+  case SEEK_CUR:
+    set = *pos;
+    break;
+  case SEEK_SET:
+    if ((err = scatp_begin(&set, pos->scat)))
+      return (int) err;
+    break;
+  case SEEK_END:
+    if ((err = scatp_end(&set, pos->scat)))
+      return (int) err;
+    break;
+  default:
+    return EINVAL;
+  }
+  if (offset >= 0) {
+    if ((err = scatp_jforward(&set, offset)) != offset)
+      return err < 0 ? (int) err : -1;
+  } else {
+    offset = -offset;
+    if ((err = scatp_jbackward(&set, offset)) != offset)
+      return err < 0 ? (int) err : -1;
+  }
+  *pos = set;
+  return 0;
+}
+
+long scatp_cpy0(const scatp *dst, const scatp *src, long num_bytes) 
+{
+  return scatp_adv_cpy0((scatp*) dst, (scatp*) src, num_bytes, 0, 0);
+}
+
+long scatp_cpy1(char *dst, const scatp *src, long num_bytes) 
+{
+  scatter dscat;
+  scatp dscatp;
+  long err;
+
+  dscat.num_elements    = 1;
+  dscat.elements[0].len = num_bytes;
+  dscat.elements[0].buf = dst;
+
+  err = scatp_begin(&dscatp, &dscat);
+  assert(err == 0);
+
+  return scatp_cpy0(&dscatp, src, num_bytes);
+}
+
+long scatp_cpy2(const scatp *dst, char *src, long num_bytes) 
+{
+  scatter sscat;
+  scatp sscatp;
+  long err;
+
+  sscat.num_elements    = 1;
+  sscat.elements[0].len = num_bytes;
+  sscat.elements[0].buf = src;
+
+  err = scatp_begin(&sscatp, &sscat);
+  assert(err == 0);
+
+  return scatp_cpy0(dst, &sscatp, num_bytes);
+}
+
+long scatp_adv_cpy0(scatp *dst, scatp *src, long num_bytes, int adv_dst, int adv_src) 
+{
+  scatter *dscat = dst->scat, *sscat = src->scat;
+  long dst_elem, src_elem, bytes_left, copy_size, dst_left, src_left;
+  char *dst_curr, *dst_end, *src_curr, *src_end;
+
+  if (scatp_is_not_legal(dst) || scatp_is_not_legal(src)) {
+    printf("illegal scatp! dst: %d src: %d\n", scatp_is_not_legal(dst), scatp_is_not_legal(src));
+    return ILLEGAL_MESSAGE;
+  }
+  if (num_bytes < 0)
+    return ILLEGAL_SERVICE;
+
+  if (scatp_is_end(dst) || scatp_is_end(src))
+    return 0;
+
+  dst_elem = dst->elem_ind;
+  dst_curr = dscat->elements[dst->elem_ind].buf + dst->buff_ind;
+  dst_end  = dscat->elements[dst->elem_ind].buf + dscat->elements[dst->elem_ind].len;
+
+  src_elem = src->elem_ind;
+  src_curr = sscat->elements[src->elem_ind].buf + src->buff_ind;
+  src_end  = sscat->elements[src->elem_ind].buf + sscat->elements[src->elem_ind].len;
+
+  bytes_left = num_bytes;
+
+  while (dst_elem < dscat->num_elements && src_elem < sscat->num_elements && bytes_left) {
+    dst_left  = dst_end - dst_curr;
+    src_left  = src_end - src_curr;
+    copy_size = (dst_left < src_left) ? dst_left : src_left;
+    copy_size = (copy_size < bytes_left) ? copy_size : bytes_left;
+
+    if (copy_size < 0) {       /* ensure no empty or negative copies */
+      printf("scatp_cpy: buffer size negative!\n");
+      return ILLEGAL_MESSAGE;
+    }
+
+    memcpy(dst_curr, src_curr, copy_size);
+    bytes_left -= copy_size;
+
+    if (copy_size != dst_left)
+      dst_curr += copy_size;
+    else {
+      while (++dst_elem < dscat->num_elements && dscat->elements[dst_elem].len == 0);
+      if (dst_elem < dscat->num_elements) {
+	dst_curr = dscat->elements[dst_elem].buf;
+	dst_end  = dscat->elements[dst_elem].buf + dscat->elements[dst_elem].len;
+      }
+    }
+    if (copy_size != src_left)
+      src_curr += copy_size;
+    else {
+      while (++src_elem < sscat->num_elements && sscat->elements[src_elem].len == 0);
+      if (src_elem < sscat->num_elements) {
+	src_curr = sscat->elements[src_elem].buf;
+	src_end  = sscat->elements[src_elem].buf + sscat->elements[src_elem].len;
+      }
+    }
+  }
+  if (bytes_left != 0) /* couldn't do the entire copy */
+    return num_bytes - bytes_left;
+
+  /* success! now, update the scatp's if requested to */
+  if (adv_dst) {
+    dst->elem_ind = dst_elem;
+    if (dst_elem != dscat->num_elements)
+      dst->buff_ind = dst_curr - dscat->elements[dst_elem].buf;
+    else
+      dst->buff_ind = 0;
+  }
+  if (adv_src) {
+    src->elem_ind = src_elem;
+    if (src_elem != sscat->num_elements)
+      src->buff_ind = src_curr - sscat->elements[src_elem].buf;
+    else
+      src->buff_ind = 0;
+  }
+  return num_bytes;
+}
+
+long scatp_adv_cpy1(char **dst, scatp *src, long num_bytes, int adv_dst, int adv_src) 
+{
+  scatter dscat;
+  scatp dscatp;
+  long ret;
+
+  dscat.num_elements    = 1;
+  dscat.elements[0].len = num_bytes;
+  dscat.elements[0].buf = *dst;
+
+  ret = scatp_begin(&dscatp, &dscat);
+  assert(ret == 0);
+
+  if ((ret = scatp_adv_cpy0(&dscatp, src, num_bytes, 0, adv_src)) == num_bytes && adv_dst)
+    *dst += num_bytes;
+
+  return ret;
+}
+
+long scatp_adv_cpy2(scatp *dst, char **src, long num_bytes, int adv_dst, int adv_src) 
+{
+  scatter sscat;
+  scatp sscatp;
+  long ret;
+
+  sscat.num_elements    = 1;
+  sscat.elements[0].len = num_bytes;
+  sscat.elements[0].buf = *src;
+
+  ret = scatp_begin(&sscatp, &sscat);
+  assert(ret == 0);
+
+  if ((ret = scatp_adv_cpy0(dst, &sscatp, num_bytes, adv_dst, 0)) == num_bytes && adv_src)
+    *src += num_bytes;
+
+  return ret;
+}
+

Added: trunk/flush/scatp.h
===================================================================
--- trunk/flush/scatp.h	2005-07-28 14:50:56 UTC (rev 252)
+++ trunk/flush/scatp.h	2005-07-31 10:51:40 UTC (rev 253)
@@ -0,0 +1,78 @@
+/*
+ * The contents of this file are subject to the FLUSH SPREAD Non-Commercial 
+ * License, Version 1.0 (the ``License''); you may not use this file except 
+ * in compliance with the License. You may obtain a copy of the License at:
+ *
+ * http://www.cnds.jhu.edu/research/group/flush_spread/FLUSH_LICENSE
+ *
+ * or in the file ``FLUSH_LICENSE'' found in the root of this distribution.
+ *
+ * Software distributed under the License is distributed on an AS IS basis, 
+ * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License 
+ * for the specific language governing rights and limitations under the 
+ * License.
+ *
+ * The Original Software is:
+ *    The Flush Spread Library
+ *     
+ * The Initial Developers of the Original Software are:
+ *    Yair Amir, John Schultz and Jonathan Stanton
+ *
+ *    All Rights Reserved.
+ *
+ */
+
+#ifndef scatp_h_2000_05_21_19_14_47_jschultz_at_cnds_jhu_edu
+#define scatp_h_2000_05_21_19_14_47_jschultz_at_cnds_jhu_edu
+
+#include <sp.h>
+#include <stdio.h>
+
+/* don't try to initialize this structure by yourself, instead call
+   scatp_begin, scatp_set, or scatp_end -- there are special cases
+   where the initial values are not obvious 
+*/
+
+typedef struct scatp  /* represents a logical position within a scatter structure */
+{ 
+  scatter *scat;
+  long elem_ind;      /* use signed because scatter uses signed on some archs */
+  long buff_ind;
+
+} scatp;
+
+long scat_capacity(const scatter *scat);
+
+/* initializers: need to call one to init a scatp properly */
+
+int scatp_begin(scatp *pos, const scatter *scat);
+int scatp_end(scatp *pos, const scatter *scat);
+int scatp_set(scatp *pos, const scatter *scat, long offset, int whence);
+
+/* some information about a scatp */
+
+int  scatp_is_legal(const scatp *pos);
+int  scatp_is_not_legal(const scatp *pos);
+int  scatp_is_end(const scatp *pos);
+int  scatp_equals(const scatp *pos1, const scatp *pos2);
+long scatp_comp(const scatp *pos1, const scatp *pos2);
+
+/* move current position of a scatp */
+
+long scatp_jforward(scatp *pos, long num_bytes);
+long scatp_jbackward(scatp *pos, long num_bytes);
+int  scatp_seek(scatp *pos, long offset, int whence);
+
+/* copy from a src to a dst like memcpy does */
+
+long scatp_cpy0(const scatp *dst, const scatp *src, long num_bytes);
+long scatp_cpy1(char *dst, const scatp *src, long num_bytes);
+long scatp_cpy2(const scatp *dst, char *src, long num_bytes);
+
+/* copy from a src to a dst like memcpy does, and advance the positions if so requested */
+
+long scatp_adv_cpy0(scatp *dst, scatp *src, long num_bytes, int adv_dst, int adv_src);
+long scatp_adv_cpy1(char **dst, scatp *src, long num_bytes, int adv_dst, int adv_src);
+long scatp_adv_cpy2(scatp *dst, char **src, long num_bytes, int adv_dst, int adv_src);
+
+#endif

Added: trunk/flush/sp_time_memb.c
===================================================================
--- trunk/flush/sp_time_memb.c	2005-07-28 14:50:56 UTC (rev 252)
+++ trunk/flush/sp_time_memb.c	2005-07-31 10:51:40 UTC (rev 253)
@@ -0,0 +1,253 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <math.h>
+#include <limits.h>
+#include <float.h>
+
+#include <sys/time.h>
+#include <sys/resource.h>
+#include <unistd.h>
+
+#include "fl.h"
+#include "stats.h"
+
+#define MY_MAX_NUM_GROUPS 1000
+#define MY_MAX_MESS_SIZE 102400
+
+enum { LOAD_MEMBER = 0, DELTA = 1, DIE_MESS = 2048 };
+
+mailbox mbox;
+service service_type;
+char    sender[MAX_GROUP_NAME];
+int     num_groups;
+char    groups[MY_MAX_NUM_GROUPS][MAX_GROUP_NAME];
+int16   mess_type;
+int     endian_mismatch;
+int     mess_len;
+char    mess[MY_MAX_MESS_SIZE];
+int     more_messes;
+
+char    daemon_name[MAX_GROUP_NAME] = "4803 at localhost";
+char    user_name[MAX_GROUP_NAME]   = "1";
+char    priv_name[MAX_GROUP_NAME];
+char    group_name[MAX_GROUP_NAME]  = "test";
+
+char *exe = 0;             /* executable's name */
+int user_type = -1;
+int num_joins_leaves = 0;  /* i: tmp var, num_joins_leaves: # of memberships to cause */
+int about_to_die = 0;      /* boolean: am I dieing after next membership? */
+int num_members;
+
+int should_sleep = 1;      /* should there be sleeps between memberships? ususally yes */
+int pretty_print = 1;      /* should the output be verbose and human readable? */
+
+stats_results sp_join_stats, sp_leave_stats, sp_cmbo_stats;
+
+int i, err, num_joins;
+
+static int printUsage(FILE *outstream) {
+  return fprintf(outstream, 
+		 "Usage: %s\r\n"
+		 "\t[-S <group size>]                  : group size (for stats; also login name, defaults to 1)\r\n"
+		 "\t[-s <address>]                     : spread daemon name - either port or port at machine\r\n"
+		 "\t[-g <group name>]                  : group name to join\r\n"
+		 "\t[-t <user type> <# joins/leaves>]  : type of user "
+		 "(LOAD_MEMBER = %d, DELTA = %d) + # of join/leave events\r\n"
+		 "\t[-f]                               : don't sleep between memberships (default is to sleep)\r\n"
+		 "\t[-r]                               : print raw scores w/ no pretty headings\r\n", 
+		 exe, LOAD_MEMBER, DELTA);
+}
+
+static void usage(int argc, char **argv) {
+  for (++argv, --argc; argc > 0; ++argv, --argc) {
+    if (!strcmp(*argv, "-S") && --argc) {
+      strncpy(user_name, *++argv, MAX_GROUP_NAME);
+
+    } else if (!strcmp(*argv, "-s") && --argc) {
+      strncpy(daemon_name, *++argv, MAX_GROUP_NAME);
+
+    } else if (!strcmp(*argv, "-g") && --argc) {
+      strncpy(group_name, *++argv, MAX_GROUP_NAME);
+
+    } else if (!strcmp(*argv, "-t") && (argc -= 2) > 0) {
+      user_type = atoi(*++argv);
+      num_joins_leaves = atoi(*++argv); 
+
+    } else if (!strcmp(*argv, "-f")) {
+      should_sleep = 0;
+
+    } else if (!strcmp(*argv, "-r")) {
+      pretty_print = 0;
+
+    } else {
+      fprintf(stderr, "Unknown cmd line param: %s\r\n", *argv);
+      exit(printUsage(stderr));
+    }
+  }
+  num_members = atoi(user_name);
+}
+
+int main(int argc, char **argv) {
+  exe = *argv;
+
+  usage(argc, argv);
+  
+  if ((err = SP_connect(daemon_name, user_name, 0, 1, &mbox, priv_name)) != ACCEPT_SESSION) {
+    fprintf(stderr, "SP_connect failure: ");
+    SP_error(err);
+    exit(1);
+  }  
+  if (user_type == DELTA) {
+    double *sp_join_times  = (double*) malloc(sizeof(double) * num_joins_leaves);
+    double *sp_leave_times = (double*) malloc(sizeof(double) * num_joins_leaves);
+    double *sp_cmbo_times  = (double*) malloc(sizeof(double) * num_joins_leaves);
+
+    if (!sp_join_times || !sp_leave_times || !sp_cmbo_times) {
+      exit(fprintf(stderr, "Couldn't mallocate tracking arrays!\r\n"));
+    }
+
+    for (i = 0; i < num_joins_leaves; ++i) {
+      double t = get_time_timeofday();
+
+      if ((err = SP_join(mbox, group_name)) < 0) {
+	fprintf(stderr, "SP_join failure: ");
+	SP_error(err);
+	exit(1);
+      }	
+      do { 
+	if ((mess_len = SP_receive(mbox, &service_type, sender, MY_MAX_NUM_GROUPS, 
+				   &num_groups, groups, &mess_type, &endian_mismatch, 
+				   MY_MAX_MESS_SIZE, mess)) < 0) {
+	  fprintf(stderr, "SP_receive failure: ");
+	  SP_error(mess_len);
+	  exit(1);
+	}
+      } while (!Is_reg_memb_mess(service_type)); 
+
+      sp_join_times[i] = get_time_timeofday() - t;
+
+      if (should_sleep) {
+	/* sleep for 4 times how long the join membership took */
+	/* allows membership to stabilize */
+	usleep((unsigned long) (4 * sp_join_times[i] * 1000));
+      }
+      
+      /* send a kill message if we are done */
+      if (i == num_joins_leaves - 1) {
+	if ((mess_len = SP_multicast(mbox, SAFE_MESS, group_name, DIE_MESS, 0, 0)) < 0) {
+	  fprintf(stderr, "SP_multicast failure: ");
+	  SP_error(err);
+	  exit(1);
+	}
+      }      
+      t = get_time_timeofday();
+
+      if ((err = SP_leave(mbox, group_name)) < 0) {
+	fprintf(stderr, "SP_leave failure: ");
+	SP_error(err);
+	exit(1); 
+      }	
+      do {
+	if ((mess_len = SP_receive(mbox, &service_type, sender, MY_MAX_NUM_GROUPS, 
+				   &num_groups, groups, &mess_type, &endian_mismatch, 
+				   MY_MAX_MESS_SIZE, mess)) < 0) {
+	  fprintf(stderr, "SP_receive failure: ");
+	  SP_error(mess_len);
+	  exit(1);
+	}
+      } while (!Is_self_leave(service_type));
+
+      sp_leave_times[i] = get_time_timeofday() - t;
+      sp_cmbo_times[i]  = sp_join_times[i] + sp_leave_times[i];
+
+      if (should_sleep) {
+	/* sleep for 4 times how long the join membership took */
+	/* allows membership to stabilize */
+	usleep((unsigned long) (4 * sp_join_times[i] * 1000));
+      }
+    }
+    /* compute statistics */
+    comp_stats(&sp_join_stats, sp_join_times, num_joins_leaves);
+    comp_stats(&sp_leave_stats, sp_leave_times, num_joins_leaves);
+    comp_stats(&sp_cmbo_stats, sp_cmbo_times, num_joins_leaves);
+
+    /* output statistics */
+    if (pretty_print) { 
+      printf("Spread Membership Timings: Group Size: %d, # Joins/Leaves: %d\r\n\r\n", 
+	     num_members, num_joins_leaves);
+
+      printf("\tSpread Join: ind. total (90%%): %.6fms, ind. total: %.6fms\r\n", 
+	     sp_join_stats.total90, sp_join_stats.total);
+
+      printf("\tSpread Leave: ind. total (90%%): %.6fms, ind. total: %.6fms\r\n", 
+	     sp_leave_stats.total90, sp_leave_stats.total);
+
+      printf("\tSpread Join/Leave: ind. total (90%%): %.6fms, ind. total: %.6fms\r\n", 
+	     sp_cmbo_stats.total90, sp_cmbo_stats.total);
+
+      printf("\r\n\t\tGroup Size\t# Trials\tQ1\tMEDIAN\tQ3"
+	     "\tMIN (5%%)\tMEAN90\tSTDDEV90\tMAX (95%%)"
+	     "\tMIN\tMEAN\tSTDDEV\tMAX\r\n");
+    }
+    printf("\tSP_Join:\t%d\t%d\t%.6f\t%.6f\t%.6f\t%.6f\t%.6f\t%.6f\t%.6f\t%.6f\t%.6f\t%.6f\t%.6f\r\n",
+	   num_members, num_joins_leaves, sp_join_stats.quart1, sp_join_stats.median,
+	   sp_join_stats.quart3, sp_join_stats.min5, sp_join_stats.mean90, 
+	   sp_join_stats.stddev90, sp_join_stats.max95, sp_join_stats.min, 
+	   sp_join_stats.mean, sp_join_stats.stddev, sp_join_stats.max);
+
+    printf("\tSP_Leave:\t%d\t%d\t%.6f\t%.6f\t%.6f\t%.6f\t%.6f\t%.6f\t%.6f\t%.6f\t%.6f\t%.6f\t%.6f\r\n",
+	   num_members, num_joins_leaves, sp_leave_stats.quart1, sp_leave_stats.median,
+	   sp_leave_stats.quart3, sp_leave_stats.min5, sp_leave_stats.mean90, 
+	   sp_leave_stats.stddev90, sp_leave_stats.max95, sp_leave_stats.min, 
+	   sp_leave_stats.mean, sp_leave_stats.stddev, sp_leave_stats.max);
+
+    printf("\tSP_Join + SP_Leave\t%d\t%d\t%.6f\t%.6f\t%.6f\t%.6f\t%.6f\t%.6f\t%.6f\t%.6f\t%.6f\t%.6f\t%.6f\r\n",
+	   num_members, num_joins_leaves, sp_cmbo_stats.quart1, sp_cmbo_stats.median,
+	   sp_cmbo_stats.quart3, sp_cmbo_stats.min5, sp_cmbo_stats.mean90, 
+	   sp_cmbo_stats.stddev90, sp_cmbo_stats.max95, sp_cmbo_stats.min, 
+	   sp_cmbo_stats.mean, sp_cmbo_stats.stddev, sp_cmbo_stats.max);
+
+  } else if (user_type == LOAD_MEMBER) { 
+    if ((err = SP_join(mbox, group_name)) < 0) {
+      fprintf(stderr, "SP_join failure: ");
+      SP_error(err);
+      exit(1);
+    }      
+    while (1) {
+      if ((mess_len = SP_receive(mbox, &service_type, sender,
+				 MY_MAX_NUM_GROUPS, &num_groups, groups,
+				 &mess_type, &endian_mismatch,
+				 MY_MAX_MESS_SIZE, mess)) < 0) { 
+	fprintf(stderr, "SP_receive failure: ");
+	SP_error(mess_len);
+	exit(1);
+      }
+      if (Is_reg_memb_mess(service_type)) {
+	if (Is_caused_leave_mess(service_type)) {
+	  if (about_to_die) {
+	    break;
+	  }
+	} else if (!Is_caused_join_mess(service_type)) {
+	  exit(fprintf(stderr, "Unexpected membership type: %d\n", service_type));
+	}
+      } else if (Is_safe_mess(service_type) && mess_type == DIE_MESS) {
+	about_to_die = 1;
+      }
+    }
+    printf("Success!\r\n");
+
+  } else { 
+    fprintf(stderr, "Unknown user type: %d\n", user_type); 
+    exit(printUsage(stderr));
+  }
+  
+  if ((err = SP_disconnect(mbox)) < 0) {
+    fprintf(stderr, "SP_disconnect failure: ");
+    SP_error(err);
+    exit(1);
+  }
+  return 0;
+}
+
+  

Added: trunk/flush/stats.c
===================================================================
--- trunk/flush/stats.c	2005-07-28 14:50:56 UTC (rev 252)
+++ trunk/flush/stats.c	2005-07-31 10:51:40 UTC (rev 253)
@@ -0,0 +1,81 @@
+#include <stdlib.h>
+#include <math.h>
+#include <sys/time.h>
+
+#include "stats.h"
+
+static int dbl_cmp(const void *arg1, const void *arg2) 
+{
+  double a1 = *(double*) arg1, a2 = *(double*) arg2;
+
+  if (a1 < a2) {
+    return -1;
+
+  } else if (a1 > a2) {
+    return 1;
+  }
+  return 0;
+}
+
+int comp_stats(stats_results *ans, double *samples, size_t num_samples) 
+{
+  size_t ind5 = num_samples / 20, ind95 = 19 * num_samples / 20, num_samples90 = ind95 - ind5 + 1;
+  size_t i;
+
+  /* total90, mean90, stddev90, min5, max95 are all computed from samples [5%, 95%] range */
+
+  if (num_samples == 0) { 
+    return -1;
+  }  
+  qsort(samples, num_samples, sizeof(double), dbl_cmp);  
+
+  ans->min    = samples[0];
+  ans->min5   = samples[ind5];
+  ans->max95  = samples[ind95];
+  ans->max    = samples[num_samples - 1];
+
+  ans->quart1 = samples[num_samples / 4];
+  ans->median = samples[num_samples / 2];
+  ans->quart3 = samples[3 * num_samples / 4];
+
+  for (i = 0, ans->total = 0, ans->total90 = 0; i < ind5; ++i) {
+    ans->total += samples[i];
+  }
+  for (i = ind5; i <= ind95; ++i) {
+    ans->total90 += samples[i];
+  }
+  for (i = ind95 + 1; i < num_samples; ++i) {
+    ans->total += samples[i];
+  }
+  ans->mean90 = ans->total90 / num_samples90;
+  ans->mean   = (ans->total + ans->total90) / num_samples;
+
+  if (num_samples != 1) {
+    for (i = 0, ans->stddev = 0, ans->stddev90 = 0; i < ind5; ++i) {
+      ans->stddev += (ans->mean - samples[i]) * (ans->mean - samples[i]);    
+    }
+    for (i = ind5; i <= ind95; ++i) {
+      ans->stddev90 += (ans->mean90 - samples[i]) * (ans->mean90 - samples[i]);
+      ans->stddev   += (ans->mean - samples[i]) * (ans->mean - samples[i]);    
+    }
+    for (i = ind95 + 1; i < num_samples; ++i) {
+      ans->stddev += (ans->mean - samples[i]) * (ans->mean - samples[i]);    
+    }
+    ans->stddev90 = sqrt(ans->stddev90 / (num_samples90 - 1));
+    ans->stddev   = sqrt(ans->stddev / (num_samples - 1));
+
+  } else {
+    ans->stddev90 = 0;
+    ans->stddev   = 0;
+  }
+  return 0;
+}
+
+double get_time_timeofday(void) 
+{
+  struct timeval used;
+
+  gettimeofday(&used, 0);
+
+  return used.tv_sec * 1000.0 + used.tv_usec / 1000.0;
+}

Added: trunk/flush/stats.h
===================================================================
--- trunk/flush/stats.h	2005-07-28 14:50:56 UTC (rev 252)
+++ trunk/flush/stats.h	2005-07-31 10:51:40 UTC (rev 253)
@@ -0,0 +1,19 @@
+#ifndef stats_h_2001_07_06_14_19_22_jschultz_at_dfusion_net
+#define stats_h_2001_07_06_14_19_22_jschultz_at_dfusion_net
+
+#include <stddef.h>
+
+typedef struct {
+  double total, total90;
+  double mean, mean90;
+  double stddev, stddev90;
+  double min, min5, max, max95;
+  double quart1, median, quart3;
+
+} stats_results;
+
+int comp_stats(stats_results *ans, double *samples, size_t num_samples);
+
+double get_time_timeofday(void);
+
+#endif

Added: trunk/flush/user.c
===================================================================
--- trunk/flush/user.c	2005-07-28 14:50:56 UTC (rev 252)
+++ trunk/flush/user.c	2005-07-31 10:51:40 UTC (rev 253)
@@ -0,0 +1,423 @@
+/*
+ * The contents of this file are subject to the FLUSH SPREAD Non-Commercial 
+ * License, Version 1.0 (the ``License''); you may not use this file except 
+ * in compliance with the License. You may obtain a copy of the License at:
+ *
+ * http://www.cnds.jhu.edu/research/group/flush_spread/FLUSH_LICENSE
+ *
+ * or in the file ``FLUSH_LICENSE'' found in the root of this distribution.
+ *
+ * Software distributed under the License is distributed on an AS IS basis, 
+ * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License 
+ * for the specific language governing rights and limitations under the 
+ * License.
+ *
+ * The Original Software is:
+ *    The Flush Spread Library
+ *     
+ * The Initial Developers of the Original Software are:
+ *    Yair Amir, John Schultz and Jonathan Stanton
+ *
+ *    All Rights Reserved.
+ *
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <fl.h>
+
+#ifdef USE_DMALLOC
+#include <dmalloc.h>
+#endif
+
+static	char	User[80];
+static  char    Spread_name[80];
+static  char    Private_group[MAX_GROUP_NAME];
+static  mailbox Mbox;
+static	int	Num_sent;
+static	int	Previous_len;
+static  int     To_exit = 0;
+
+static	void	Print_menu();
+static	void	User_command();
+static	void	Read_message();
+static	void	Usage( int argc, char *argv[] );
+static  void	Bye();
+
+int main(int argc, char *argv[]) {
+  int ret, major, minor, patch;
+
+  Usage(argc, argv);
+
+  FL_version(&major, &minor, &patch);
+  printf("Flush library version is %d.%d.%d\n", major, minor, patch);
+
+  ret = FL_connect( Spread_name, User, LOW_PRIORITY, &Mbox, Private_group );
+  if( ret < 0 ) {
+    FL_error( ret );
+    Bye();
+  }
+  printf("User: connected to %s with private group %s\n", Spread_name, Private_group );
+ 
+  Print_menu();
+
+  printf("\nUser> ");
+  fflush(stdout);
+
+  Num_sent = 0;
+
+  for(;;)
+    User_command();
+  
+  return 0;
+}
+
+static	void	User_command()
+{
+  char	command[130];
+  char	mess[102400];
+  char	group[80];
+  char	groups[10][MAX_GROUP_NAME];
+  int	num_groups;
+  int	mess_len;
+  int	ret;
+  int	i;
+
+  for (i = 0; i < sizeof(command); i++) 
+    command[i] = 0;
+
+  if (fgets(command, 130, stdin) == 0) 
+    Bye();
+
+  switch(command[0]) {
+  case 'j':
+    ret = sscanf( &command[2], "%s", group );
+    if( ret < 1 ) {
+      printf(" invalid group \n");
+      break;
+    }
+    ret = FL_join( Mbox, group );
+    if( ret < 0 ) FL_error( ret );
+    break;
+
+    case 'l':
+      ret = sscanf( &command[2], "%s", group );
+      if( ret < 1 ) 
+	{
+	  printf(" invalid group \n");
+	  break;
+	}
+      ret = FL_leave( Mbox, group );
+      if( ret < 0 ) FL_error( ret );
+
+      break;
+
+  case 'g': {
+    int i;
+
+    if ((num_groups = sscanf(&command[2], "%s", groups[0])) < 1) {
+      printf("invalid group\n");
+      break;
+    }
+    printf("enter number of receivers: ");
+    fflush(stdout);
+    if (fgets(mess, 200, stdin) == 0 || 
+	sscanf(mess, "%d", &num_groups) != 1 || 
+	num_groups < 1 || num_groups > 9) {
+      printf("invalid number of receivers, must be an integer in the range [1, 9]\n");
+      break;
+    }
+    for (i = 1; i <= num_groups; ++i) {
+      printf("enter recvr %d: ", i);
+      fflush(stdout);
+      if (fgets(mess, MAX_GROUP_NAME, stdin) == 0 || sscanf(mess, "%s", groups[i]) != 1) 
+	Bye();
+    }
+    printf("enter message: ");
+    fflush(stdout);
+    if (fgets(mess, 200, stdin) == 0) Bye();
+    mess_len = strlen(mess);
+
+    printf("user.c: subgroupcasting message of size %d to group %s with receivers:\n", mess_len, groups[0]);
+    for (i = 1; i <= num_groups; ++i)
+      printf("%s\n", groups[i]);
+
+    if ((ret = FL_subgroupcast(Mbox, SAFE_MESS, groups[0], num_groups, groups+1, 0, mess_len, mess)) != mess_len) {
+      if (ret < 0) {
+	FL_error(ret);
+	Bye();
+      } else
+	printf("Wierd return from FL_subgroupcast %d, should be %d\n", ret, mess_len);
+    }
+    Num_sent++;
+    break;
+  }
+
+    case 's':
+      num_groups = sscanf(&command[2], "%s", groups[0]);
+      if( num_groups < 1 ) 
+	{
+	  printf(" invalid group \n");
+	  break;
+	}
+      printf("enter message: ");
+      ret = (int) fgets( mess, 200, stdin );
+      if( ret==0 ) Bye();
+      mess_len = strlen( mess );
+
+      printf("user.c : multicasting message of size %d:\n'%s'\n", mess_len, mess);
+
+      ret = FL_multicast( Mbox, SAFE_MESS, groups[0], 1, mess_len, mess );
+      if( ret < 0 ) 
+	{
+	  FL_error( ret );
+	  Bye();
+	}
+      Num_sent++;
+      break;
+
+    case 'b':
+      ret = sscanf( &command[2], "%s", group );
+      if( ret != 1 ) strcpy( group, "dummy_group_name" );
+      printf("enter size of each message: ");
+      ret = (int) fgets( mess, 200, stdin );
+      if( ret==0 ) Bye();
+      ret = sscanf(mess, "%d", &mess_len );
+      if( ret !=1 ) mess_len = Previous_len;
+      if( mess_len < 0 ) mess_len = 0;
+      Previous_len = mess_len;
+      printf("sending 10 messages of %d bytes\n", mess_len );
+      for( i=0; i<10; i++ )
+	{
+	  Num_sent++;
+	  sprintf( mess, "mess num %d", Num_sent );
+	  ret= FL_multicast( Mbox, FIFO_MESS, group, 2, mess_len, mess );
+
+	  if( ret < 0 ) 
+	    {
+	      FL_error( ret );
+	      Bye();
+	    }
+	  printf("sent message %d (total %d)\n", i+1, Num_sent );
+	}
+      break;
+    case 'r':
+
+      Read_message();
+      break;
+
+    case 'p':
+
+      ret = FL_poll( Mbox );
+      printf("Polling says: %d\n", ret );
+      break;
+
+      /*
+	case 'e':
+
+	E_attach_fd( Mbox, Read_message, 0, HIGH_PRIORITY );
+	break;
+
+	case 'd':
+
+	E_detach_fd( Mbox );
+	break;
+
+      */
+    case 'f':
+      ret = sscanf( &command[2], "%s", group);
+      if( ret < 1 ) 
+	{
+	  printf(" invalid group \n");
+	  break;
+	}
+      ret = FL_flush(Mbox, group);
+
+      printf("Sent a FLUSH_OK message to group '%s'\n", group);
+
+      if (ret < 0) {
+	FL_error(ret);
+	Bye();
+      }
+      break;
+
+    case 'q':
+      Bye();
+      break;
+
+    default:
+      printf("\nUnknown commnad\n");
+      Print_menu();
+
+      break;
+    }
+  printf("\nUser> ");
+  fflush(stdout);
+}
+
+static	void	Print_menu()
+{
+  printf("\n");
+  printf("==========\n");
+  printf("User Menu:\n");
+  printf("----------\n");
+  printf("\n");
+  printf("\tj <group> -- join a group\n");
+  printf("\tl <group> -- leave a group\n");
+  printf("\n");
+  printf("\ts <group> -- send a message\n");
+  printf("\tg <group> -- send a subgroup message\n");
+  printf("\tb <group> -- send a burst of messages\n");
+  printf("\n");
+  printf("\tr -- receive a message (stuck) \n");
+  printf("\tp -- poll for a message \n");
+  /*
+    printf("\te -- enable asynchonous read (default)\n");
+    printf("\td -- disable asynchronous read \n");
+  */
+  printf("\tf <group> -- send a FLUSH_OK message\n");
+  printf("\n");
+  printf("\tq -- quit\n");
+  fflush(stdout);
+}
+
+static	void	Read_message()
+{
+
+  static	char		mess[102400];
+  char		sender[MAX_GROUP_NAME];
+  char		target_groups[100][MAX_GROUP_NAME];
+  group_id	*grp_id;
+  int32		*num_vs;
+  char		*vs_members;
+  int		num_groups;
+  int		num_bytes;
+  int		service_type;
+  int16		mess_type;
+  int		endian_mismatch;
+  int		i;
+  int		ret;
+  int             more_messes;
+
+  ret = FL_receive( Mbox, &service_type, sender, 100, &num_groups, target_groups, 
+		    &mess_type, &endian_mismatch, sizeof(mess), mess, &more_messes);
+  printf("\n============================\n");
+  if( ret < 0 ) 
+    {
+      if( ! To_exit )
+	{
+	  FL_error( ret );
+	  printf("\n============================\n");
+	  printf("\nBye.\n");
+	}
+      exit( 0 );
+    }
+  if( Is_regular_mess( service_type ) )
+    {
+      mess[ret] = 0;
+      if     ( Is_unreliable_mess( service_type ) ) printf("received UNRELIABLE ");
+      else if( Is_reliable_mess(   service_type ) ) printf("received RELIABLE ");
+      else if( Is_fifo_mess(       service_type ) ) printf("received FIFO ");
+      else if( Is_causal_mess(     service_type ) ) printf("received CAUSAL ");
+      else if( Is_agreed_mess(     service_type ) ) printf("received AGREED ");
+      else if( Is_safe_mess(       service_type ) ) printf("received SAFE ");
+
+      printf("message from %s, of type %d, (endian %d) to %d groups \n(%d bytes): %s\n",
+	     sender, mess_type, endian_mismatch, num_groups, ret, mess );
+      printf("Message is:\n'%s'\n", mess);
+      if (Is_subgroup_mess(service_type)) {
+	int i;
+
+	printf("Received a subgroupcast message to group %s: receivers:\n", target_groups[num_groups - 1]);
+	for (i = 0; i < num_groups - 1; ++i)
+	  printf("\t%s\n", target_groups[i]);
+      }
+    } else if( Is_membership_mess( service_type ) ){
+      if     ( Is_reg_memb_mess( service_type ) )
+	{
+	  num_bytes = 0;
+	  grp_id = (group_id *)&mess[num_bytes];
+	  num_bytes += sizeof( group_id );
+	  num_vs = (int32 *)&mess[num_bytes];
+	  num_bytes += sizeof( int32 );
+	  vs_members = &mess[num_bytes];
+
+	  printf("Received REGULAR membership for group %s with %d members, where I am member %d:\n",
+		 sender, num_groups, mess_type );
+
+	  for( i=0; i < num_groups; i++ )
+	    printf("\t%s\n", &target_groups[i][0]);
+
+	  printf("grp id is %d %d %d\n", grp_id->id[0], grp_id->id[1], grp_id->id[2]);
+
+	  if( Is_caused_join_mess( service_type ) )
+	    {
+	      printf("Due to the JOIN of %s\n", vs_members );
+	    }else if( Is_caused_leave_mess( service_type ) ){
+	      printf("Due to the LEAVE of %s\n", vs_members);
+	    }else if( Is_caused_disconnect_mess( service_type ) ){
+	      printf("Due to the DISCONNECT of %s\n", vs_members );
+	    }else if( Is_caused_network_mess( service_type ) ){
+	      printf("Due to NETWORK change. ");
+	      printf("VS set has %d members:\n", *num_vs );
+	      for( i=0; i < *num_vs; i++, vs_members+= MAX_GROUP_NAME )
+		printf("\t%s\n", vs_members );
+	    }
+	}else if( Is_transition_mess(   service_type ) ) {
+	  printf("received TRANSITIONAL membership for group %s\n", sender );
+	}else if( Is_caused_leave_mess( service_type ) ){
+	  printf("received membership message that left group %s\n", sender );
+	}else printf("received incorrecty membership message of type %d\n", service_type );
+    }else if (Is_flush_req_mess(service_type)) {
+      printf("received a FLUSH_REQ message for group %s\n", sender);
+    }else printf("received message of unknown message type %d with ret %d\n", service_type, ret);
+
+  printf("There are %d buffered messages waiting to be delivered\n", more_messes);
+
+  printf("\n");
+  printf("User> ");
+  fflush(stdout);
+}
+
+static	void	Usage(int argc, char *argv[])
+{
+
+  sprintf( User, "user" );
+  sprintf( Spread_name, "3333 at localhost");
+  while( --argc > 0 )
+    {
+      argv++;
+
+      if( !strncmp( *argv, "-u", 2 ) )
+	{
+	  strcpy( User, argv[1] );
+	  argc--; argv++;
+	}else if( !strncmp( *argv, "-s", 2 ) ){
+	  strcpy( Spread_name, argv[1] ); 
+	  argc--; argv++;
+	}else{
+	  printf( "Usage: user\n%s\n%s\n",
+		  "\t[-u <user name>]  : unique (in this machine) user name",
+		  "\t[-s <address>]    : either port or port at machine");
+	  exit( 0 );
+	}
+    }
+}
+
+static  void	Bye()
+{
+  To_exit = 1;
+
+  printf("\nBye.\n");
+
+  printf("Calling FL_disconnect(mbox = %d)\n", Mbox);
+
+  FL_disconnect( Mbox );
+  exit( 0 );
+}
+
+
+
+
+

Added: trunk/include/fl.h
===================================================================
--- trunk/include/fl.h	2005-07-28 14:50:56 UTC (rev 252)
+++ trunk/include/fl.h	2005-07-31 10:51:40 UTC (rev 253)
@@ -0,0 +1,114 @@
+/*
+ * The contents of this file are subject to the FLUSH SPREAD Non-Commercial 
+ * License, Version 1.0 (the ``License''); you may not use this file except 
+ * in compliance with the License. You may obtain a copy of the License at:
+ *
+ * http://www.cnds.jhu.edu/research/group/flush_spread/FLUSH_LICENSE
+ *
+ * or in the file ``FLUSH_LICENSE'' found in the root of this distribution.
+ *
+ * Software distributed under the License is distributed on an AS IS basis, 
+ * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License 
+ * for the specific language governing rights and limitations under the 
+ * License.
+ *
+ * The Original Software is:
+ *    The Flush Spread Library
+ *     
+ * The Initial Developers of the Original Software are:
+ *    Yair Amir, John Schultz and Jonathan Stanton
+ *
+ *    All Rights Reserved.
+ *
+ */
+
+#ifndef fl_h_2000_03_20_14_36_26_jschultz_at_cnds_jhu_edu
+#define fl_h_2000_03_20_14_36_26_jschultz_at_cnds_jhu_edu
+
+#include <sp.h>
+
+/* FL service types */
+#define DONT_BLOCK     0x10000000  
+#define FLUSH_REQ_MESS 0x20000000
+#define SUBGROUP_CAST  0x40000000
+
+/* FL service query macros */
+#define Is_flush_req_mess(serv) (((serv) & FLUSH_REQ_MESS) != 0)
+#define Is_subgroup_mess(serv)  (((serv) & SUBGROUP_CAST) != 0)
+
+/* FL error codes */
+#define ILLEGAL_PARAM        -24
+#define WOULD_BLOCK          -25
+#define ILLEGAL_MESSAGE_TYPE -26
+#define ILLEGAL_STATE        -27
+#define ILLEGAL_RECEIVERS    -28
+
+/* maximum # of usable scatter elements for FL msgs */
+#define FL_MAX_SCATTER_ELEMENTS (MAX_CLIENT_SCATTER_ELEMENTS - 1)
+
+/* minimum message type a user is allowed to use - less than this is illegal */
+#define FL_MIN_LEGAL_MESS_TYPE ((int16) -32765)
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+int   FL_lib_init(void);
+
+void  FL_version(int *major_ver, int *minor_ver, int *patch_ver);
+
+int   FL_connect(const char *daemon_name, const char *user_name, int priority, 
+		 mailbox *mbox, char *private_name);
+
+int   FL_disconnect(mailbox mbox);
+
+int   FL_join(mailbox mbox, const char *group_name);
+
+int   FL_leave(mailbox mbox, const char *group_name);
+
+int   FL_flush(mailbox mbox, const char *group_name);
+
+int   FL_unicast(mailbox mbox, service serv_type, const char *group_name, 
+		 const char *recvr_name, int16 mess_type, int mess_len, const char *mess);
+
+int   FL_scat_unicast(mailbox mbox, service serv_type, const char *group_name,
+		      const char *recvr_name, int16 mess_type, const scatter *scat);
+
+int   FL_subgroupcast(mailbox mbox, service serv_type, const char *group_name,
+		      int num_recvrs, char recvr_names[][MAX_GROUP_NAME],
+		      int16 mess_type, int mess_len, const char *mess);
+
+int   FL_scat_subgroupcast(mailbox mbox, service serv_type, const char *group_name,
+			   int num_recvrs, char recvr_names[][MAX_GROUP_NAME],
+			   int16 mess_type, const scatter *scat);
+
+int   FL_multicast(mailbox mbox, service serv_type, const char *group_name,
+		   int16 mess_type, int mess_len, const char *mess);
+
+int   FL_scat_multicast(mailbox mbox, service serv_type, const char *group_name,
+			int16 mess_type, const scatter *scat_mess);
+
+int   FL_receive(mailbox mbox, service *serv_type, char *sender_name,
+		 int max_groups, int *num_groups, char group_names[][MAX_GROUP_NAME],
+		 int16 *mess_type, int *endian_mismatch, int max_mess_len,
+		 char *mess, int *more_msgs);
+  
+int   FL_scat_receive(mailbox mbox, service *serv_type, char *sender_name,
+		      int max_groups, int *num_groups, char group_names[][MAX_GROUP_NAME],
+		      int16 *mess_type, int *endian_mismatch, scatter *scat_mess, 
+		      int *more_msgs);
+  
+int   FL_more_msgs(mailbox mbox);
+
+int   FL_poll(mailbox mbox);
+
+void  FL_error(int error_code);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
+
+
+




More information about the Spread-cvs mailing list