[Spread-cvs] commit: r253 - in trunk: . daemon docs docs/flush
docs/flush/docs docs/flush/man flush include
jschultz at spread.org
jschultz at spread.org
Sun Jul 31 06:51:41 EDT 2005
Author: jschultz
Date: 2005-07-31 06:51:40 -0400 (Sun, 31 Jul 2005)
New Revision: 253
Added:
trunk/docs/flush/
trunk/docs/flush/docs/
trunk/docs/flush/docs/FL_connect.html
trunk/docs/flush/docs/FL_disconnect.html
trunk/docs/flush/docs/FL_error.html
trunk/docs/flush/docs/FL_flush.html
trunk/docs/flush/docs/FL_join.html
trunk/docs/flush/docs/FL_leave.html
trunk/docs/flush/docs/FL_more_msgs.html
trunk/docs/flush/docs/FL_multicast.html
trunk/docs/flush/docs/FL_poll.html
trunk/docs/flush/docs/FL_receive.html
trunk/docs/flush/docs/FL_version.html
trunk/docs/flush/docs/access.html
trunk/docs/flush/docs/error.html
trunk/docs/flush/docs/flush_spread_title.gif
trunk/docs/flush/docs/functions.html
trunk/docs/flush/docs/index.html
trunk/docs/flush/docs/messages.html
trunk/docs/flush/docs/misc.html
trunk/docs/flush/index.html
trunk/docs/flush/man/
trunk/docs/flush/man/FL_connect.3
trunk/docs/flush/man/FL_disconnect.3
trunk/docs/flush/man/FL_error.3
trunk/docs/flush/man/FL_flush.3
trunk/docs/flush/man/FL_join.3
trunk/docs/flush/man/FL_leave.3
trunk/docs/flush/man/FL_more_msgs.3
trunk/docs/flush/man/FL_multicast.3
trunk/docs/flush/man/FL_poll.3
trunk/docs/flush/man/FL_receive.3
trunk/docs/flush/man/FL_scat_multicast.3
trunk/docs/flush/man/FL_scat_receive.3
trunk/docs/flush/man/FL_scat_subgroupcast.3
trunk/docs/flush/man/FL_scat_unicast.3
trunk/docs/flush/man/FL_subgroupcast.3
trunk/docs/flush/man/FL_unicast.3
trunk/docs/flush/man/FL_version.3
trunk/flush/
trunk/flush/Makefile.in
trunk/flush/configure.in
trunk/flush/fl.c
trunk/flush/fl_p.h
trunk/flush/fl_time_memb.c
trunk/flush/scatp.c
trunk/flush/scatp.h
trunk/flush/sp_time_memb.c
trunk/flush/stats.c
trunk/flush/stats.h
trunk/flush/user.c
trunk/include/fl.h
Modified:
trunk/daemon/groups.c
Log:
Added in flush code (still needs to conform to new Spread semantics) as best I thought appropriate. Minor comments added to groups.c.
Modified: trunk/daemon/groups.c
===================================================================
--- trunk/daemon/groups.c 2005-07-28 14:50:56 UTC (rev 252)
+++ trunk/daemon/groups.c 2005-07-31 10:51:40 UTC (rev 253)
@@ -2280,13 +2280,13 @@
litend_ptr = litbox_ptr + 1;
}
- /* NOTE: this code assumes that grp->mboxes contains no duplicates */
+ /* NOTE: the following code assumes that grp->mboxes contains no duplicates */
if (num_groups == 1) { /* no need to do extra copy work if only one group -> just use litbox 'array' directly */
break;
}
- if (num_mbox != 0) {
+ if (num_mbox != 0) { /* mboxes contains some entries already */
bigend_ptr = mboxes + num_mbox;
for (; litbox_ptr != litend_ptr; ++litbox_ptr)
@@ -2314,7 +2314,7 @@
bigbox_ptr = litbox_ptr;
bigend_ptr = litend_ptr;
- } else {
+ } else { /* otherwise use unique entries we've built in mboxes */
bigbox_ptr = mboxes;
bigend_ptr = mboxes + num_mbox;
}
Added: trunk/docs/flush/docs/FL_connect.html
===================================================================
--- trunk/docs/flush/docs/FL_connect.html 2005-07-28 14:50:56 UTC (rev 252)
+++ trunk/docs/flush/docs/FL_connect.html 2005-07-31 10:51:40 UTC (rev 253)
@@ -0,0 +1,166 @@
+<HTML><HEAD><TITLE>Manpage of FL_connect</TITLE>
+</HEAD>
+
+<BODY bgcolor="#FFFFFF">
+
+<!--#include virtual="/includes/header-a" -->
+
+<a href="http://www.cnds.jhu.edu/research/group/flush_spread">
+<img src="flush_spread_title.gif" alt="FLUSH SPREAD" border=0>
+</a>
+
+<!--#include virtual="/includes/header-b" -->
+
+<H1>FL_connect</H1>
+Section: User Manuals (3)<BR>Updated: Dec 2000<BR>
+<HR>
+
+<H2>NAME</H2>
+
+FL_connect - connect an application to a Spread daemon using Flush Spread semantics.
+
+<H2>SYNOPSIS</H2>
+
+<B>#include <fl.h></B>
+
+<P>
+<B>int FL_connect(const char *</B><I>daemon_name</I><B>, const char *</B><I>user_name</I><B>, int </B><I>priority</I><B>, int </B><I>group_membership</I><B>, mailbox *</B><I>mbox</I><B>, char *</B><I>private_name</I><B>);</B>
+
+<H2>DESCRIPTION</H2>
+
+<B>FL_connect</B>
+
+is the initial call an application must make to establish a
+connection with a Spread daemon. All other Flush Spread calls
+refer to a valid
+<I>mbox </I>
+
+returned by this function.
+<P>
+The
+<I>daemon_name</I>
+
+is the name of the Spread daemon with which to connect. It should be a
+string in one of the following forms:
+<DL COMPACT><DT><DD>
+<DL COMPACT>
+<DT><B>4803</B>
+
+<DD>
+This will connect to the Spread daemon on the local
+machine running on port 4803. This form cannot be
+used to connect to a Windows95/NT machine.</p>
+
+<DT><B>4803 at localhost</B>
+
+<DD>
+This will also connect to the Spread daemon
+on the local machine running on port 4803.
+This form can be used on Windows95/NT machines.</p>
+
+<DT><B>4803 at host.domain.edu</B> or <B>4803 at 128.220.221.99</B>
+
+<DD>
+This will connect to the machine identified by either
+domain name or IP address at the specified port.</p>
+</DL></DL>
+
+The
+<I>user_name </I>
+
+is the name this connection would like to be known as. It must be
+unique on the machine running the spread daemon. The name can be an
+arbitrary length string with the same character restrictions as a
+group name (mainly it cannot contain the '#' character).
+<P>
+<I>priority</I>
+
+is a boolean (non-zero true, zero false) for whether this connection
+will be a "priority" connection or not. Currently this has no effect.
+<P>
+<I>group_membership</I>
+
+is a boolean for whether this connection will receive group membership
+messages or not. Usually for Flush Spread applications this parameter
+should be true. If your application doesn't need group membership
+messages, then Spread may provide the semantics that you need more
+efficiently than Flush Spread.
+<P>
+The
+<I>mbox</I>
+
+should be a pointer to a mailbox variable. After the
+<B>FL_connect</B>
+
+call successfully returns, this variable will hold the valid mbox for
+this new connection.
+<P>
+The
+<I>private_name</I>
+
+should be a pointer to a string big enough to hold at least
+MAX_GROUP_NAME characters. After the call returns it will contain the
+private group name of this connection. These group names are what are
+reported in membership messages and can be used to send unicast and
+subgroup-multicast messages to this connection. No other applications
+can join this special group.
+<P>
+<H2>RETURN VALUES</H2>
+
+Returns
+<B>ACCEPT_SESSION </B>
+
+on success or one of the following errors ( < 0 ):
+<DL COMPACT>
+
+<DT><B>ILLEGAL_SPREAD</B>
+<DD>
+The
+<I>daemon_name</I>
+given to connect to was illegal for some reason. Usually because
+it was a unix socket on Windows95/NT, an improper format for a host
+or an illegal port number. </p>
+
+<DT><B>COULD_NOT_CONNECT</B>
+
+<DD>
+Lower level socket calls failed to allow a connection to the
+specified spread daemon right now.</p>
+
+<DT><B>CONNECTION_CLOSED</B>
+
+<DD>
+During communication to establish the connection errors occured
+and the setup could not be completed.</p>
+
+<DT><B>REJECT_VERSION</B>
+
+<DD>
+The daemon and/or libraries have a version mismatch.</p>
+
+<DT><B>REJECT_NO_NAME</B>
+
+<DD>
+No user name was provided.</p>
+
+<DT><B>REJECT_ILLEGAL_NAME</B>
+
+<DD>
+Name provided violated some requirement (length or used an illegal character).</p>
+
+<DT><B>REJECT_NOT_UNIQUE</B>
+
+<DD>
+Name provided is not unique on this daemon. Recommended response is to try
+again with a different name.
+</DL>
+
+<H2>AUTHOR</H2>
+
+John Schultz <<A HREF="mailto:jschultz at cnds.jhu.edu">jschultz at cnds.jhu.edu</A>>
+<P>
+
+<!--#include virtual="/includes/footer" -->
+
+</BODY>
+</HTML>
Added: trunk/docs/flush/docs/FL_disconnect.html
===================================================================
--- trunk/docs/flush/docs/FL_disconnect.html 2005-07-28 14:50:56 UTC (rev 252)
+++ trunk/docs/flush/docs/FL_disconnect.html 2005-07-31 10:51:40 UTC (rev 253)
@@ -0,0 +1,59 @@
+<HTML><HEAD><TITLE>Manpage of FL_disconnect</TITLE>
+</HEAD>
+
+<body bgcolor="#FFFFFF">
+
+<!--#include virtual="/includes/header-a" -->
+
+<a href="http://www.cnds.jhu.edu/research/group/flush_spread">
+<img src="flush_spread_title.gif" alt="FLUSH SPREAD" border=0>
+</a>
+
+<!--#include virtual="/includes/header-b" -->
+
+<H1>FL_disconnect</H1>
+Section: User Manuals (3)<BR>Updated: Dec 2000<BR>
+<HR>
+
+<H2>NAME</H2>
+
+FL_disconnect - destroy a Flush Spread connection between an application and a Spread daemon.
+
+<H2>SYNOPSIS</H2>
+
+<B>#include <fl.h></B>
+
+<P>
+<B>int FL_disconnect(mailbox </B><I>mbox</I><B>);</B>
+
+<H2>DESCRIPTION</H2>
+
+<B>FL_disconnect</B>
+
+should be called when the application is finished
+with a connection to a Spread daemon. The application may have
+other connections still open to the daemon and may open a new
+connection after disconnecting. Any data that was available on
+the connection will be lost after the call returns.
+<P>
+The
+<I>mbox </I>
+
+should be for the connection you wish to close.
+<H2>RETURN VALUES</H2>
+
+Returns 0 on success or
+<B>ILLEGAL_SESSION</B>
+
+when the
+<I>mbox</I>
+
+given does not represent a valid connection.
+<H2>AUTHOR</H2>
+
+John Schultz <<A HREF="mailto:jschultz at cnds.jhu.edu">jschultz at cnds.jhu.edu</A>>
+
+<!--#include virtual="/includes/footer" -->
+
+</BODY>
+</HTML>
Added: trunk/docs/flush/docs/FL_error.html
===================================================================
--- trunk/docs/flush/docs/FL_error.html 2005-07-28 14:50:56 UTC (rev 252)
+++ trunk/docs/flush/docs/FL_error.html 2005-07-31 10:51:40 UTC (rev 253)
@@ -0,0 +1,42 @@
+<HTML><HEAD><TITLE>Manpage of FL_error</TITLE>
+</HEAD>
+
+<body bgcolor="#FFFFFF">
+
+<!--#include virtual="/includes/header-a" -->
+
+<a href="http://www.cnds.jhu.edu/research/group/flush_spread">
+<img src="flush_spread_title.gif" alt="FLUSH SPREAD" border=0>
+</a>
+
+<!--#include virtual="/includes/header-b" -->
+
+<H1>FL_error</H1>
+Section: User Manuals (3)<BR>Updated: Dec 2000<BR>
+<HR>
+
+<H2>NAME</H2>
+
+FL_error - Flush Spread error string reporting
+<H2>SYNOPSIS</H2>
+
+<B>#include <fl.h></B>
+
+<P>
+<B>void FL_error(int </B><I>error_code</I><B>);</B>
+
+<H2>DESCRIPTION</H2>
+
+<B>FL_error</B>
+
+prints an error string based on the Flush Spread or Spread error code
+<I>error_code</I>.
+
+<H2>AUTHOR</H2>
+
+John Schultz <<A HREF="mailto:jschultz at cnds.jhu.edu">jschultz at cnds.jhu.edu</A>>
+
+<!--#include virtual="/includes/footer" -->
+
+</BODY>
+</HTML>
Added: trunk/docs/flush/docs/FL_flush.html
===================================================================
--- trunk/docs/flush/docs/FL_flush.html 2005-07-28 14:50:56 UTC (rev 252)
+++ trunk/docs/flush/docs/FL_flush.html 2005-07-31 10:51:40 UTC (rev 253)
@@ -0,0 +1,136 @@
+<HTML><HEAD><TITLE>Manpage of FL_flush</TITLE>
+</HEAD>
+
+<body bgcolor="#FFFFFF">
+
+<!--#include virtual="/includes/header-a" -->
+
+<a href="http://www.cnds.jhu.edu/research/group/flush_spread">
+<img src="flush_spread_title.gif" alt="FLUSH SPREAD" border=0>
+</a>
+
+<!--#include virtual="/includes/header-b" -->
+
+<H1>FL_flush</H1>
+Section: User Manuals (3)<BR>Updated: Dec 2000<BR>
+<HR>
+
+<H2>NAME</H2>
+
+FL_flush - flush a group in response to receiving a flush request message.
+<H2>SYNOPSIS</H2>
+
+<B>#include <fl.h></B>
+
+<P>
+<B>int FL_flush(mailbox </B><I>mbox</I><B>, const char *</B><I>group_name</I><B>);</B>
+
+<H2>DESCRIPTION</H2>
+
+<B>FL_flush</B>
+
+flushes the group named
+<I>group_name</I>
+
+in which the connection represented by
+<I>mbox</I>
+
+has received a flush request message. A flush request message is
+delivered in a group when the underlying membership of that group
+changes; it is a signal to the application that the membership has
+changed and some of the original members may have "gone away" (meaning
+that they are no longer in the group with this connection). However,
+Flush Spread will not install a new membership in the group until each
+of the members of the new membership flush the group. Flushing a
+group gives Flush Spread the permission of this connection to go ahead
+and install a new view/membership for the group.
+<P>
+During the time after receiving a flush request message but before
+flushing the group the application must be _VERY_ careful about
+calling receive functions; if the application isn't careful enough it
+can permantently block the entire group. For example, say the
+underlying membership of a group of 10 members changed such that this
+connection is now alone in the group (there is no way an application
+can detect the make-up of a group after receiving a flush request).
+If the application simply calls receive before flushing the group and
+hasn't ensured that it will receive a message somehow (see below) it
+will permanently block itself in the receive and any other members
+that might later be added to that group.
+<P>
+There are a couple ways to avoid this problem, which isn't a bug by
+the way :P, (1) don't call receive functions after receiving a flush
+request message and before flushing the group, (2) call receive but
+set the DONT_BLOCK service flag; if there aren't any messages to
+receive it will break out with a WOULD_BLOCK error, (3) ensure that
+there will be a message to receive somehow. There are several ways to
+do this, but one 100% sure way to do this is if the application sends
+or has sent a message on this connection (with the SELF_DISCARD
+service _NOT_ set) that it has not yet received back. In this case,
+the application will eventually receive its own message (even if it
+was to another group) on the connection and can therefore be assured
+of not blocking permanently.
+<P>
+The application _IS_ allowed to send messages to a group after
+receiving a flush request for that group and before flushing the
+group. However, only a subset of the original members of the current
+view will receive these messages (note that I said a subset, not a
+strict subset). Technically, this is always the case in Spread: an
+application can determine which other members received certain
+messages for sure by (1) application level message receipt
+acknowledgement, (2) employing the safety properties of SAFE messages
+(although this doesn't guarantee that those members actually processed
+and handled the message), (3) employing the virtual synchrony property
+and the transitional set of the new membership when it is installed
+(although, again, this doesn't guarantee that those members actually
+processed and handled the messages). See FL_receive or read up on
+group communication for more in-depth discussions of this matter.
+<P>
+Anyways, when the application is ready it can flush the group. Once
+it flushes the group it is not allowed to send any messages to that
+group, until it receives the new membership for that group. If the
+application breaks this rule, it will receive an ILLEGAL_STATE error.
+<P>
+<H2>RETURN VALUES</H2>
+
+Returns 0 on success or one of the following errors ( < 0 ):
+<DL COMPACT>
+<DT><B>ILLEGAL_SESSION</B>
+
+<DD>
+The connection represented by
+<I>mbox</I>
+
+is illegal, usually because it is not active.</p>
+
+<DT><B>ILLEGAL_GROUP</B>
+
+<DD>
+The
+<I>group_name</I>
+
+given to flush was illegal for some reason, usually because it was of
+length 0 or length > MAX_GROUP_NAME. This error can also be returned
+when a group is flushed for a connection that it has not yet joined or
+is already leaving.</p>
+
+<DT><B>ILLEGAL_STATE</B>
+
+<DD>
+A connection may flush a group only once in response to each flush
+request message delivered in that group. If you violate this rule you
+will get an ILLEGAL_STATE error.</p>
+
+<DT><B>CONNECTION_CLOSED</B>
+
+<DD>
+Errors occurred during communication and the flush could not be
+initiated.
+</DL>
+<H2>AUTHOR</H2>
+
+John Schultz <<A HREF="mailto:jschultz at cnds.jhu.edu">jschultz at cnds.jhu.edu</A>>
+
+<!--#include virtual="/includes/footer" -->
+
+</BODY>
+</HTML>
Added: trunk/docs/flush/docs/FL_join.html
===================================================================
--- trunk/docs/flush/docs/FL_join.html 2005-07-28 14:50:56 UTC (rev 252)
+++ trunk/docs/flush/docs/FL_join.html 2005-07-31 10:51:40 UTC (rev 253)
@@ -0,0 +1,77 @@
+<HTML><HEAD><TITLE>Manpage of FL_join</TITLE>
+</HEAD>
+
+<body bgcolor="#FFFFFF">
+
+<!--#include virtual="/includes/header-a" -->
+
+<a href="http://www.cnds.jhu.edu/research/group/flush_spread">
+<img src="flush_spread_title.gif" alt="FLUSH SPREAD" border=0>
+</a>
+
+<!--#include virtual="/includes/header-b" -->
+
+<H1>FL_join</H1>
+Section: User Manuals (3)<BR>Updated: Dec 2000<BR>
+<HR>
+
+<H2>NAME</H2>
+
+FL_join - join a connection to a Flush Spread group
+<H2>SYNOPSIS</H2>
+
+<B>#include <fl.h></B>
+
+<P>
+<B>int FL_join(mailbox </B><I>mbox</I><B>, const char *</B><I>group_name</I><B>);</B>
+
+<H2>DESCRIPTION</H2>
+
+<B>FL_join</B>
+
+joins the connection represented by
+<I>mbox</I>
+
+to a group with the name
+<I>group_name</I>.
+
+If the group does not exist among the Spread daemons it is created,
+otherwise the connection is joined to the existing group. A
+connection may not join a group it is already a member of, which it
+has already joined, or from which it is currently leaving.
+<H2>RETURN VALUES</H2>
+
+Returns 0 on success or one of the following errors ( < 0 ):
+<DL COMPACT>
+<DT><B>ILLEGAL_GROUP</B>
+
+<DD>
+The
+<I>group_name</I>
+
+given to join was illegal for some reason, usually because it was of
+length 0 or length > MAX_GROUP_NAME. This error is also returned if a
+group with which this connection is already involved (i.e. - already
+joining, already joined, currently leaving) is joined again.</p>
+
+<DT><B>ILLEGAL_SESSION</B>
+
+<DD>
+The connection represented by
+<I>mbox</I>
+
+is illegal, usually because it is not active.</p>
+<DT><B>CONNECTION_CLOSED</B>
+
+<DD>
+Errors occurred during communication and the join could not be
+initiated.</p>
+</DL>
+<H2>AUTHOR</H2>
+
+John Schultz <<A HREF="mailto:jschultz at cnds.jhu.edu">jschultz at cnds.jhu.edu</A>>
+
+<!--#include virtual="/includes/footer" -->
+
+</BODY>
+</HTML>
Added: trunk/docs/flush/docs/FL_leave.html
===================================================================
--- trunk/docs/flush/docs/FL_leave.html 2005-07-28 14:50:56 UTC (rev 252)
+++ trunk/docs/flush/docs/FL_leave.html 2005-07-31 10:51:40 UTC (rev 253)
@@ -0,0 +1,82 @@
+<HTML><HEAD><TITLE>Manpage of FL_leave</TITLE>
+</HEAD>
+
+<body bgcolor="#FFFFFF">
+
+<!--#include virtual="/includes/header-a" -->
+
+<a href="http://www.cnds.jhu.edu/research/group/flush_spread">
+<img src="flush_spread_title.gif" alt="FLUSH SPREAD" border=0>
+</a>
+
+<!--#include virtual="/includes/header-b" -->
+
+<H1>FL_leave</H1>
+Section: User Manuals (3)<BR>Updated: Dec 2000<BR>
+<HR>
+
+<H2>NAME</H2>
+
+FL_leave - remove a connection from a Flush Spread group
+<A NAME="lbAC"> </A>
+<H2>SYNOPSIS</H2>
+
+<B>#include <fl.h></B>
+
+<P>
+<B>int FL_leave(mailbox </B><I>mbox</I><B>, const char *</B><I>group_name</I><B>);</B>
+
+<H2>DESCRIPTION</H2>
+
+<B>FL_leave</B>
+
+removes the connection represented by
+<I>mbox</I>
+
+from a Flush Spread group with the name
+<I>group_name</I>.
+
+A connection may only leave a group after it has been installed as a
+member of the group by being included in a Flush Spread
+view/membership. Also a connection may not leave a group if it has
+already requested to leave that group and the self-leave message has
+not yet been received.
+<H2>RETURN VALUES</H2>
+
+Returns 0 on success or one of the following errors ( < 0 ):
+<DL COMPACT>
+<DT><B>ILLEGAL_GROUP</B>
+
+<DD>
+The
+<I>group_name</I>
+
+given to leave was illegal for some reason. Usually because it was of
+length 0 or length > MAX_GROUP_NAME. This error is also returned if
+the connection was not a full-fledged member (i.e. - not yet included
+in a Flush membership message for that group, or already leaving) of
+the group.</p>
+
+<DT><B>ILLEGAL_SESSION</B>
+
+<DD>
+The connection represented by
+<I>mbox</I>
+
+is illegal. Usually because it is not active.</p>
+
+<DT><B>CONNECTION_CLOSED</B>
+
+<DD>
+Errors occurred during communication
+and the leave could not be initiated.
+</DL>
+
+<H2>AUTHOR</H2>
+
+John Schultz <<A HREF="mailto:jschultz at cnds.jhu.edu">jschultz at cnds.jhu.edu</A>>
+
+<!--#include virtual="/includes/footer" -->
+
+</BODY>
+</HTML>
Added: trunk/docs/flush/docs/FL_more_msgs.html
===================================================================
--- trunk/docs/flush/docs/FL_more_msgs.html 2005-07-28 14:50:56 UTC (rev 252)
+++ trunk/docs/flush/docs/FL_more_msgs.html 2005-07-31 10:51:40 UTC (rev 253)
@@ -0,0 +1,79 @@
+<HTML><HEAD><TITLE>Manpage of FL_more_msgs</TITLE>
+</HEAD>
+
+<body bgcolor="#FFFFFF">
+
+<!--#include virtual="/includes/header-a" -->
+
+<a href="http://www.cnds.jhu.edu/research/group/flush_spread">
+<img src="flush_spread_title.gif" alt="FLUSH SPREAD" border=0>
+</a>
+
+<!--#include virtual="/includes/header-b" -->
+
+<H1>FL_more_msgs</H1>
+Section: User Manuals (3)<BR>Updated: Dec 2000<BR>
+<HR>
+
+<H2>NAME</H2>
+
+FL_more_msgs - returns the number of complete messages buffered and
+ready to be received on a connection.
+<H2>SYNOPSIS</H2>
+
+<B>#include <fl.h></B>
+
+<P>
+<B>int FL_more_msgs(mailbox </B><I>mbox</I><B>);</B>
+
+<H2>DESCRIPTION</H2>
+
+<B>FL_more_msgs</B>
+
+allows an application to check to see if any complete messages are
+already buffered and ready to be received on the connection represented by
+<I>mbox</I>.
+
+<P>
+<B>NOTE</B>,
+
+that this function
+<B>CANNOT </B>
+
+be used as an I/O polling function to check if a receive call should
+be done. If this function returns zero there still might be a message
+ready to receive on the connection: FL_poll, and file descriptor
+selects/polls can detect if there is activity (not necessarily a
+message) on a connection, while DONT_BLOCK receive semantics can
+detect whether or not there is a message on the connection and
+FL_more_msgs can only answer if there are any buffered messages
+already on the connection.
+<P>
+This function is merely a helper function to re-get the current status
+of the
+<I>more_messes</I>
+
+parameter returned from a call to
+<B>FL_receive</B>.
+
+<H2>RETURN VALUES</H2>
+
+Returns the number of complete buffered messages ready to be received,
+or one of the following errors ( < 0):
+<DL COMPACT>
+<DT><B>ILLEGAL_SESSION</B>
+
+<DD>
+The connection represented by
+<I>mbox</I>
+
+is illegal, usually because it is not active.
+</DL>
+<H2>AUTHOR</H2>
+
+John Schultz <<A HREF="mailto:jschultz at cnds.jhu.edu">jschultz at cnds.jhu.edu</A>>
+
+<!--#include virtual="/includes/footer" -->
+
+</BODY>
+</HTML>
Added: trunk/docs/flush/docs/FL_multicast.html
===================================================================
--- trunk/docs/flush/docs/FL_multicast.html 2005-07-28 14:50:56 UTC (rev 252)
+++ trunk/docs/flush/docs/FL_multicast.html 2005-07-31 10:51:40 UTC (rev 253)
@@ -0,0 +1,242 @@
+<HTML><HEAD><TITLE>Manpage of FL_multicast</TITLE>
+</HEAD>
+
+<body bgcolor="#FFFFFF">
+
+<!--#include virtual="/includes/header-a" -->
+
+<a href="http://www.cnds.jhu.edu/research/group/flush_spread">
+<img src="flush_spread_title.gif" alt="FLUSH SPREAD" border=0>
+</a>
+
+<!--#include virtual="/includes/header-b" -->
+
+<H1>FL_multicast</H1>
+Section: User Manuals (3)<BR>Updated: Dec 2000<BR>
+<HR>
+
+<H2>NAME</H2>
+
+FL_unicast, FL_scat_unicast, FL_subgroupcast, FL_scat_subgroupcast, FL_multicast, FL_scat_multicast - multicast messages to subsets of Flush Spread groups.
+<H2>SYNOPSIS</H2>
+
+<B>#include <fl.h></B>
+
+<P>
+<B>int FL_unicast(mailbox </B><I>mbox</I><B>, service </B><I>serv_type</I><B>, const char *</B><I>group_name</I><B>, const char *</B><I>recvr_name</I><B>, int16 </B><I>mess_type</I><B>, int </B><I>mess_len</I><B>, const char *</B><I>mess</I><B>);</B>
+
+<P>
+<B>int FL_scat_unicast(mailbox </B><I>mbox</I><B>, service </B><I>serv_type</I><B>, const char *</B><I>group_name</I><B>, const char *</B><I>recvr_name</I><B>, int16 </B><I>mess_type</I><B>, const scatter *</B><I>scat</I><B>);</B>
+
+<P>
+<B>int FL_subgroupcast(mailbox </B><I>mbox</I><B>, service </B><I>serv_type</I><B>, const char *</B><I>group_name</I><B>, int </B><I>num_recvrs</I><B>, const char </B><I>recvr_names</I><B>[][MAX_GROUP_NAME], int16 </B><I>mess_type</I><B>, int </B><I>mess_len</I><B>, const char *</B><I>mess</I><B>);</B>
+
+<P>
+<B>int FL_scat_subgroupcast(mailbox </B><I>mbox</I><B>, service </B><I>serv_type</I><B>, const char *</B><I>group_name</I><B>, int </B><I>num_recvrs</I><B>, const char </B><I>recvr_names</I><B>[][MAX_GROUP_NAME], int16 </B><I>mess_type</I><B>, const char *</B><I>scat</I><B>);</B>
+
+<P>
+<B>int FL_unicast(mailbox </B><I>mbox</I><B>, service </B><I>serv_type</I><B>, const char *</B><I>group_name</I><B>, int16 </B><I>mess_type</I><B>, int </B><I>mess_len</I><B>, const char *</B><I>mess</I><B>);</B>
+
+<P>
+<B>int FL_scat_unicast(mailbox </B><I>mbox</I><B>, service </B><I>serv_type</I><B>, const char *</B><I>group_name</I><B>, int16 </B><I>mess_type</I><B>, const scatter *</B><I>scat</I><B>);</B>
+
+<H2>DESCRIPTION</H2>
+
+<B>FL_multicast</B>
+
+and its variants all multicast a message from the connection represented by
+<I>mbox</I>
+
+to a subset of the membership of the Flush Spread group named
+<I>group_name</I>.
+
+<P>
+Note, that unlike Spread, Flush Spread does not provide guarantees
+about messages sent to different groups; meaning that FIFO, CAUSAL,
+SAFE, etc. safety properties are not maintained across different Flush
+Spread groups, but only within a group to which they are sent. For
+example, if an application sends a FIFO message A to the private group
+G and then sends another FIFO message B to the regular group H, of
+which G is a member, then G could receive B before A or vice versa.
+<P>
+In order to multicast a message to a group, the destination group
+<I>group_name</I>
+
+must either be a private group name, or this connection must be a
+full-fledged member of that regular group (has joined and been
+included in a membership of the group, and is not leaving the group,
+see FL_join, FL_leave) and not be in an illegal state to multicast (by
+having recently flushed the group and not yet received its next
+view/membership, see FL_flush).
+<P>
+The
+<I>serv_type</I>
+
+is a type field that should be set to the service type this message
+requires. The valid flags for multicasting messages are:
+<P>
+<DL COMPACT><DT><DD>
+<B>UNRELIABLE_MESS</B>
+
+<BR>
+
+
+<B>RELIABLE_MESS </B>
+
+<BR>
+
+
+<B>FIFO_MESS</B>
+
+<BR>
+
+
+<B>CAUSAL_MESS</B>
+
+<BR>
+
+
+<B>AGREED_MESS</B>
+
+<BR>
+
+
+<B>SAFE_MESS</B>
+
+</DL>
+
+<P>
+This type field can be bit-wise ORed with other flags like
+SELF_DISCARD if desired. Currently SELF_DISCARD is the only
+additional flag for multicasts: SELF_DISCARD will cause the
+multicasted message to NOT be delivered back to this connection.
+<P>
+The string
+<I>group_name</I>
+
+indicates the name of the Flush Spread group to which this message
+should be sent. This group can be either a regular or a private Flush
+Spread group, as dicussed above. The multicast variants sends the
+message to all members of a group, while the unicast and subgroupcast
+variants allow the sender to specify a subset of the group to which to
+send the message. For the unicast variants the sender specifies the
+private group name of the intended recipient in the variable
+<I>recvr_name</I>.
+
+For the subgroupcast variants the sender specifies the number of
+recipients in the variable
+<I>num_recvrs</I>
+
+and a doubly scripted array of the members' private group names in the
+variable
+<I>recvr_names</I>,
+
+where duplicate names are tolerated. For the unicast and subgroupcast
+variants all specified receivers must have been members of the most
+recently installed view/membership of the group. If the application
+breaks this rule it will receive an ILLEGAL_RECEIVERS error.
+<P>
+The
+<I>mess_type</I>
+
+is a 16 bit integer which can be used by the application arbitrarily.
+However, Flush Spread restricts which values can be used to be greater
+than or equal to FL_MIN_LEGAL_MESS_TYPE. If the application sends
+with a message type of less than FL_MIN_LEGAL_MESS_TYPE, then the
+error ILLEGAL_MESSAGE_TYPE will be returned. The intent of this
+message type is that the application can use it to name different
+kinds of data messages so they can be differentiated without looking
+into the body of the message. This value
+<B>WILL</B>
+
+be endian corrected upon returning.
+<P>
+The non-scatter variants use a single buffer to pass the body of the
+message to be sent. The
+<I>mess_len</I>
+
+field gives the message length in bytes. While the
+<I>mess</I>
+
+field is a pointer to the buffer containing the message. For the
+scatter variants, both of these parameters are replaced with one
+pointer,
+<I>scat_mess</I>,
+
+to a scatter structure, which is similiar to an iovec. This allows
+messages made up of several parts to be sent without an extra copy on
+systems which support scatter-gather. However, the number of scatter
+elements is restricted to be non-negative and less than or equal to
+FL_MAX_SCATTER_ELEMENTS. If a buffer length is negative or an illegal
+number of scatter elements is used then ILLEGAL_MESSAGE will be
+returned.
+
+<H2>RETURN VALUES</H2>
+
+Returns the number of bytes sent on success or one of the following
+errors ( < 0 ):
+<DL COMPACT>
+<DT><B>ILLEGAL_SESSION</B>
+
+<DD>
+The connection represented by
+<I>mbox</I>
+
+was illegal, usually because it is no longer active.</p>
+
+<DT><B>ILLEGAL_GROUP</B>
+
+<DD>
+The
+<I>group_name</I>
+
+given to multicast to was illegal for some reason. Usually because it
+was of length 0 or length > MAX_GROUP_NAME. This error can also be
+returned if the connection was not a full-fledged member (i.e. - not
+yet included in a Flush membership message for the group, or already
+leaving) of a regular group.</p>
+
+<DT><B>ILLEGAL_RECEIVERS</B>
+
+<DD>
+A unicast or subgroupcast specified private group names that weren't
+members of
+<I>group_name</I>'s
+
+most recent view/membership.</p>
+
+<DT><B>ILLEGAL_STATE</B>
+
+<DD>
+A multicast to a group was attempted after flushing the group and
+before receiving the next view/membership for that group.</p>
+
+<DT><B>ILLEGAL_SERVICE</B>
+
+<DD>
+If the service type was using bits reserved by Flush Spread or Spread.</p>
+
+<DT><B>ILLEGAL_PARAM</B>
+
+<DD>
+An illegal parameter, such as a negative array size was passed.</p>
+
+<DT><B>ILLEGAL_MESSAGE</B>
+
+<DD>
+The message had an illegal structure, like a negative buffer size or
+an illegal number of scatter elements.</p>
+
+<DT><B>CONNECTION_CLOSED</B>
+
+<DD>
+Errors occurred during communication and the multicast could not be completed.
+</DL>
+<H2>AUTHOR</H2>
+
+John Schultz <<A HREF="mailto:jschultz at cnds.jhu.edu">jschultz at cnds.jhu.edu</A>>
+
+<!--#include virtual="/includes/footer" -->
+
+</BODY>
+</HTML>
Added: trunk/docs/flush/docs/FL_poll.html
===================================================================
--- trunk/docs/flush/docs/FL_poll.html 2005-07-28 14:50:56 UTC (rev 252)
+++ trunk/docs/flush/docs/FL_poll.html 2005-07-31 10:51:40 UTC (rev 253)
@@ -0,0 +1,68 @@
+<HTML><HEAD><TITLE>Manpage of FL_poll</TITLE>
+</HEAD>
+
+<body bgcolor="#FFFFFF">
+
+<!--#include virtual="/includes/header-a" -->
+
+<a href="http://www.cnds.jhu.edu/research/group/flush_spread">
+<img src="flush_spread_title.gif" alt="FLUSH SPREAD" border=0>
+</a>
+
+<!--#include virtual="/includes/header-b" -->
+
+<H1>FL_poll</H1>
+Section: User Manuals (3)<BR>Updated: Dec 2000<BR>
+<HR>
+
+<H2>NAME</H2>
+
+FL_poll - returns the amount, in bytes, of activity on a connection.
+<H2>SYNOPSIS</H2>
+
+<B>#include <fl.h></B>
+
+<P>
+<B>int FL_poll(mailbox </B><I>mbox</I><B>);</B>
+
+<H2>DESCRIPTION</H2>
+
+<B>FL_poll</B>
+
+is a way to poll the connection represented by
+<I>mbox</I>
+
+to see if there is any activity on that connection.
+<P>
+<B>NOTE</B>,
+
+however that activity does
+<B>NOT</B>
+
+necessarily mean that a message is available to be read, as in Spread.
+Only a call to FL_more_msgs that returns a positive number of buffered
+messages or a DONT_BLOCK receive call can check for sure to see if a
+message is now available to be read. If you don't use either of these
+semantics, then you may enter into a blocking receive call, which can
+sometimes be dangerous (see FL_flush).
+<H2>RETURN VALUES</H2>
+
+Returns the number of bytes of activity currently on the connection,
+or one of the following errors ( < 0 ):
+<DL COMPACT>
+<DT><B>ILLEGAL_SESSION</B>
+
+<DD>
+The connection represented by
+<I>mbox</I>
+
+is illegal, usually because it is not active.
+</DL>
+<H2>AUTHOR</H2>
+
+John Schultz <<A HREF="mailto:jschultz at cnds.jhu.edu">jschultz at cnds.jhu.edu</A>>
+
+<!--#include virtual="/includes/footer" -->
+
+</BODY>
+</HTML>
Added: trunk/docs/flush/docs/FL_receive.html
===================================================================
--- trunk/docs/flush/docs/FL_receive.html 2005-07-28 14:50:56 UTC (rev 252)
+++ trunk/docs/flush/docs/FL_receive.html 2005-07-31 10:51:40 UTC (rev 253)
@@ -0,0 +1,522 @@
+<HTML><HEAD><TITLE>Manpage of FL_receive</TITLE>
+</HEAD>
+
+<body bgcolor="#FFFFFF">
+
+<!--#include virtual="/includes/header-a" -->
+
+<a href="http://www.cnds.jhu.edu/research/group/flush_spread">
+<img src="flush_spread_title.gif" alt="FLUSH SPREAD" border=0>
+</a>
+
+<!--#include virtual="/includes/header-b" -->
+
+<BODY>
+<H1>FL_receive</H1>
+Section: User Manuals (3)<BR>Updated: Dec 2000<BR>
+<HR>
+
+<H2>NAME</H2>
+
+Fl_receive, FL_scat_receive - receive a message on a Flush Spread connection.
+<H2>SYNOPSIS</H2>
+
+<B>#include <fl.h></B>
+
+<P>
+<B>int FL_receive(mailbox </B><I>mbox</I><B>, service *</B><I>serv_type</I><B>, char </B><I>sender</I><B>[MAX_GROUP_NAME], int </B><I>max_groups</I><B>, int *</B><I>num_groups</I><B>, char </B><I>groups</I><B>[][MAX_GROUP_NAME], int16 *</B><I>mess_type</I><B>, int *</B><I>endian_mismatch</I><B>, int </B><I>max_mess_len</I><B>, char *</B><I>mess</I><B>, int *</B><I>more_messes</I><B>);</B>
+
+<P>
+<B>int FL_scat_receive(mailbox </B><I>mbox</I><B>, service *</B><I>serv_type</I><B>, char </B><I>sender</I><B>[MAX_GROUP_NAME], int </B><I>max_groups</I><B>, int *</B><I>num_groups</I><B>, char </B><I>groups</I><B>[][MAX_GROUP_NAME], int16 *</B><I>mess_type</I><B>, int *</B><I>endian_mismatch</I><B>, scatter *</B><I>scat_mess</I><B>, int *</B><I>more_messes</I><B>);</B>
+
+<H2>DESCRIPTION</H2>
+
+<B>FL_receive</B>
+
+is the general purpose message receipt function for Flush Spread.
+Messages for all groups joined on this connection will arrive on the
+same mailbox. A call to
+<B>FL_receive</B>
+
+will perform a receive on a connection to get a single message from
+any one of the groups to which it is joined.
+<P>
+After a call to receive completes, a number of the passed fields are
+set to values indicating meta-information about the message (such as
+destination group, message type, endianness, etc). The meanings of
+these different meta fields depends on the type of message received
+and the receipt services requested.
+<P>
+Input Parameters:
+<DL COMPACT><DT><DD>
+<DL COMPACT>
+<DT><I>mbox</I>
+
+<DD>
+Represents the connection on which to receive a message.</p>
+<DT><I>serv_type</I>
+
+<DD>
+A pointer to a service bit field, requesting some receive services.
+This field can have the DONT_BLOCK and/or DROP_RECV service bit flags
+flipped on. This bit field should be zeroed out before each call if
+no special receive service is to be requested.</p>
+<DT><I>sender</I>
+
+<DD>
+A pointer to a character array with storage for at least
+MAX_GROUP_NAME characters.</p>
+<DT><I>max_groups</I>
+
+<DD>
+A non-negative int representing how many group names the application
+is willing to receive in this receive call.</p>
+<DT><I>num_groups</I>
+
+<DD>
+A pointer to an int.</p>
+<DT><I>groups</I>
+
+<DD>
+An array of group names containing storage for at least
+<I>max_groups</I>
+
+* MAX_GROUP_NAME characters.</p>
+
+<DT><I>mess_type</I>
+
+<DD>
+A pointer to an int16.</p>
+
+<DT><I>endian_mismatch</I>
+
+<DD>
+A pointer to an int.</p>
+
+<DT><I>max_mess_len</I>
+
+<DD>
+A non-negative integer representing how many bytes of data the
+application is willing to receive in this call.</p>
+
+<DT><I>mess</I>
+
+<DD>
+A pointer to a buffer containing storage for at least
+<I>max_mess_len</I>
+
+bytes.</p>
+
+<DT><I>scat_mess</I>
+
+<DD>
+A pointer to a scatter (a cousin of an iovec) with a non-negative
+number of scatter elements that is less than or equal to
+MAX_SCATTER_ELEMENTS (that's not a typo :), and all non-negative
+buffer lengths in the indicated scatter elements to get scatter-gather
+semantics.
+</DL></DL>
+
+<P>
+Receive Semantics:
+<P>
+Normally, a call to receive will block if no messages are immediately
+available.
+<P>
+Normally, when calling a receive function if a user's buffer
+(<I>groups</I>,
+
+<I>mess</I>,
+
+or
+<I>scat_mess</I>)
+
+is too small to contain the data to be returned, then a
+GROUPS_TOO_SHORT or BUFFER_TOO_SHORT error code will be returned. In
+this case, the
+<I>serv_type</I>,
+
+and
+<I>mess_type</I>
+
+of the message are filled in from the offending message and the
+parameters
+<I>num_groups</I>,
+
+and
+<I>endian_mismatch</I>
+
+reflect information about the necessary buffers' sizes. If
+<I>max_groups</I>
+
+was big enough, then
+*<I>num_groups</I>
+
+will be zero, otherwise it will be the negative of how many group
+names are available to be received (i.e. -7 means the
+<I>max_groups</I>
+
+was less than 7 and 7 group names can be received). If
+<I>max_mess_len</I>
+
+or
+<I>scat_mess</I>
+
+was big enough then
+*<I>endian_mismatch</I>
+
+will be zero, otherwise it will be the negative of how much data is
+available to be received (i.e. -32768 means the msg buffer was too
+small and 32768 bytes of data can be received). The offending message
+is still available to be received through later calls to receive with
+appropriately sized user buffers.
+<P>
+The parameter
+<I>serv_type</I>
+
+allows the applications to request different receive services that
+affect the normal semantics of receive. Currently, the only services
+are DONT_BLOCK and DROP_RECV.
+<P>
+The DONT_BLOCK service makes a receive call non-blocking. With this
+service, the receive call will return quickly either with a message or
+with the error code WOULD_BLOCK if no message is available. Using
+this service is the only way to detect if a message is ready on the
+connection in a non-blocking manner in Flush Spread.
+<P>
+The DROP_RECV service forces Flush Spread to read the current message
+off of the connection regardless of whether or not the user's buffers
+are big enough to fit all of the data. In this case, a TOO_SHORT
+error code is still returned even though the call actually
+succeeded. Anyway, as much data as can be fit into the user's buffers
+will be stuffed into them, and that message will no longer be on the
+connection. Also,
+*<I>num_groups</I>
+
+will still be set to the negative size of how many names were
+available (if
+<I>groups</I>
+
+wasn't big enough), but
+*<I>endian_mismatch</I>
+
+will not reflect the size of the data that could have been received (if
+<I>mess</I>
+
+or
+<I>scat_mess</I>
+
+wasn't big enough), instead it indicates whether or not the sending
+machine had an opposite endianness or not. Using DROP_RECV in this
+manner, when a buffer problem occurs there is no way to determine how
+big the data portion of the message actually was. This service is
+meant to be used when a call to receive without DROP_RECV fails with a
+buffer error, but it is determined that the message isn't all that
+important, or something along those lines; otherwise reallocate your
+receive buffers appropriately and re-call receive again without
+the DROP_RECV service.
+<P>
+Output Parameters:
+<P>
+Upon a successful return from receive,
+*<I>serv_type</I>
+
+will be filled out with the message type that was just received. The
+specific type of message received can be tested using the different
+message and membership type access macros. The rest of the
+parameters' meanings differ depending on the
+<I>serv_type</I>.
+
+<P>
+Regular Messages:
+<P>
+If the
+<I>serv_type</I>
+
+is a REG_MESSAGE (i.e. a data message) then:
+<P>
+The parameter
+<I>sender</I>
+
+will be filled with the private group name of the sending connection.
+<P>
+The parameter
+<I>num_groups</I>
+
+will be set to the number of group names filled into
+<I>groups</I>.
+
+<P>
+The
+<I>groups</I>
+
+array will contain the group to which this message was sent. If the
+message is a SUBGROUP_CAST (check the service type, see FL_multicast)
+then all of the recipients' private group names will be listed and the
+last name in the array (i.e.
+groups[*<I>num_groups</I>-1])
+
+will be the regular group to which this message was sent.
+<P>
+If DROP_RECV is being used, and
+<I>groups</I>
+
+is too small, then as many names as can fit will be filled in as above
+described. If the message was a SUBGROUP_CAST, then the last group in
+the array (i.e.
+groups[<I>max_groups</I>-1])
+
+will be the destination group for the message. If
+<I>max_groups</I>
+
+is zero then even the destination group is not reported.
+<P>
+The parameter
+*<I>mess_type</I>
+
+will be set to the message type field the application sent with
+the original message, which is a restricted (see FL_multicast) 16 bit
+integer. This value is endian corrected upon returning.
+<P>
+The parameter
+*<I>endian_mismatch</I>
+
+will be set to true (non-zero) if the endianness of the sending
+machine is the opposite of this receiving machine's.
+<P>
+The actual message body being received will be filled into either the
+<I>mess</I>
+
+or
+<I>scat_mess</I>
+
+parameter.
+<P>
+Flush Request Message:
+<P>
+If this is a FLUSH_REQ_MESS then this represents that the underlying
+membership has changed and that this application needs to flush this
+group before the new view/membership can be installed (see
+FL_flush).
+<P>
+The
+<I>sender</I>
+
+will be filled in with the name of the group this connection needs
+to flush. All the other fields are set to empty values.
+<P>
+Membership Messages:
+<P>
+If this is a MEMB_MESSAGE (i.e. membership message) and it
+specifically is a REG_MEMB_MESS, then:
+<P>
+The
+<I>sender</I>
+
+will be filled with the name of the group for which the membership
+change is occuring.
+<P>
+The
+<I>mess_type</I>
+
+field will be set to the index of this process in the
+<I>groups</I>
+
+array (see below).
+<P>
+The
+<I>endian_mismatch</I>
+
+field will be set to 0 since there are no endian issues with regular memberships.
+<P>
+The
+<I>groups</I>
+
+array and message
+body are used to provide two kinds of membership information about the change that just
+occured in this group. The
+<I>num_groups</I>
+
+field will be set to the number of members in the group in the
+<B>new</B>
+
+membership (i.e. after the change occured). Correspondingly, the
+<I>groups</I>
+
+array will be filled in with the private group names of all members of this
+group in the
+<B>new</B>
+
+membership. This list of names is always in the same order at all receipients
+and thus can be used to deterministically pick a group representative if
+one is needed by the application.
+<P>
+The second set of information is stored in the message body. The data
+buffer will include the following fields:
+<DL COMPACT><DT><DD>
+<DL COMPACT>
+<DT><B>group_id vid;</B>
+
+<DD>
+<BR>
+
+<DT><B>int num_members;</B>
+
+<DD>
+<BR>
+
+<DT><B>char dgroups[][MAX_GROUP_NAME];</B>
+
+<DD>
+</DL></DL>
+
+<P>
+The vid and num_members parameters will already be endian corrected.
+The dgroups array will contain num_members group names. The content
+of the dgroups array is dependent upon the type of the membership
+change:
+<DL COMPACT><DT><DD>
+<DL COMPACT>
+<DT><B>CAUSED_BY_JOIN:</B>
+
+<DD>
+dgroups contains the private group of the joining process.</p>
+<DT><B>CAUSED_BY_LEAVE:</B>
+
+<DD>
+dgroups contains the private group of the leaving process.</p>
+<DT><B>CAUSED_BY_DISCONNECT:</B>
+
+<DD>
+dgroups contains the private group of the disconnecting process.</p>
+<DT><B>CAUSED_BY_NETWORK:</B>
+
+<DD>
+dgroups contains the group names of the members of the new membership who came
+with
+<B>this</B>
+
+application to the new view/membership. Using this data the previous
+membership listing and the new membership listing an application can
+determine all the members that left and were added in the NETWORK
+membership change. Note, that a member can both leave and be added in
+a
+<B>single</B>
+
+NETWORK change.
+</DL></DL>
+
+<P>
+Transitional Message:
+<P>
+If this is a MEMB_MESSAGE and specifically it is a TRANSITION_MESS,
+then this means that one or more of the current members of the group
+has left and not all of the safety guarantees can be met w.r.t. all of
+the original members for the messages delivered after this signal and
+before the next view/membership. Transitional messages never have a
+CAUSED_BY type because they only serve to signal up to where SAFE
+delivery and AGREED delivery (with no holes) is guaranteed in the
+view/membership in which they are delivered. Only one TRANSITION_MESS
+is delivered per group view/membership per connection.
+<P>
+The
+<I>sender</I>
+
+will be filled in with the name of the group for which the membership
+change is occurring. All the other parameters will be set to empty values.
+<P>
+Self-Leave Message:
+<P>
+If this is a MEMB_MESSAGE (i.e. membership message) and it is neither
+a REG_MEMB_MESS or a TRANSITION_MESS, then it represents exactly the
+situtation where the member receiving this message has left a group
+and this is notification that the leave has occured, thus it is
+sometimes called a self-leave message. The simplest test for this is
+if a message is CAUSED_BY_LEAVE and REG_MEMB_MESS is false (zero),
+then it is a self-leave message.
+<P>
+The other members of the group that this member just left will receive
+a normal TRANSITION_MESS, REG_MEMB_MESS pair as described above
+showing this connection leaving.
+<P>
+For self-leave messages the
+<I>sender</I>
+
+field will be filled in with the name of the group this connection is
+leaving. All the other fields are set to empty values.
+<P>
+<H2>RETURN VALUES</H2>
+
+Returns the size of the data portion of the message received on
+success or one of the following errors ( < 0 ):
+<DL COMPACT>
+<DT><B>ILLEGAL_SESSION</B>
+
+<DD>
+The connection represented by
+<I>mbox</I>
+
+was illegal, usually because it is not active.</p>
+
+<DT><B>ILLEGAL_PARAM</B>
+
+<DD>
+An illegal parameter was passed, like a negative array size.</p>
+
+<DT><B>ILLEGAL_MESSAGE</B>
+
+<DD>
+The msg buffer had an illegal structure, like an illegal number of
+scatter elements or a negatively sized buffer.</p>
+
+<DT><B>WOULD_BLOCK</B>
+
+<DD>
+The receive call was made with the DONT_BLOCK service and it would
+have blocked because no messages were ready to be received.</p>
+
+<DT><B>GROUPS_TOO_SHORT</B>
+
+<DD>
+The receive call was made with a groups array that was too small.
+*<I>num_groups</I>
+
+is filled in with the negative number of names that could have been
+returned. Note, that the msg buffer could also be too short, and this
+should be determined by examining
+<I>endian_mismatch</I>
+
+as described above. If DROP_RECV was used this error code actually
+represents success (see above).</p>
+
+<DT><B>BUFFER_TOO_SHORT</B>
+
+<DD>
+The receive call was made with a msg buffer that was too small.
+*<I>endian_mismatch</I>
+
+is filled in with the negative number of bytes that could have been
+received if DROP_RECV isn't used. If it is used then this error code
+actually represents success and
+<I>endian_mismatch</I>
+
+has the normal meaning (see above). Note, that the groups buffer
+could also be too short, this should be determined by examining
+<I>num_groups</I>
+
+as described above.</p>
+
+<DT><B>CONNECTION_CLOSED</B>
+
+<DD>
+During communication to receive, communication errors occurred
+and the receive could not be completed.
+</DL>
+<H2>AUTHOR</H2>
+
+John Schultz <<A HREF="mailto:jschultz at cnds.jhu.edu">jschultz at cnds.jhu.edu</A>>
+
+<!--#include virtual="/includes/footer" -->
+
+</BODY>
+</HTML>
Added: trunk/docs/flush/docs/FL_version.html
===================================================================
--- trunk/docs/flush/docs/FL_version.html 2005-07-28 14:50:56 UTC (rev 252)
+++ trunk/docs/flush/docs/FL_version.html 2005-07-31 10:51:40 UTC (rev 253)
@@ -0,0 +1,44 @@
+<HTML><HEAD><TITLE>Manpage of FL_version</TITLE>
+</HEAD>
+
+<body bgcolor="#FFFFFF">
+
+<!--#include virtual="/includes/header-a" -->
+
+<a href="http://www.cnds.jhu.edu/research/group/flush_spread">
+<img src="flush_spread_title.gif" alt="FLUSH SPREAD" border=0>
+</a>
+
+<!--#include virtual="/includes/header-b" -->
+
+<H1>FL_version</H1>
+Section: User Manuals (3)<BR>Updated: Dec 2000<BR>
+<HR>
+
+<H2>NAME</H2>
+
+FL_version - returns library version information.
+<A NAME="lbAC"> </A>
+<H2>SYNOPSIS</H2>
+
+<B>#include <fl.h></B>
+
+<P>
+<B>void FL_version(int *</B><I>major_ver</I><B>, int *</B><I>minor_ver</I><B>, int *</B><I>patch_ver</I><B>);</B>
+
+<A NAME="lbAD"> </A>
+<H2>DESCRIPTION</H2>
+
+<B>FL_version</B>
+
+returns the major, minor, and patch version numbers of the Flush
+Spread library in use.
+<A NAME="lbAE"> </A>
+<H2>AUTHOR</H2>
+
+John Schultz <<A HREF="mailto:jschultz at cnds.jhu.edu">jschultz at cnds.jhu.edu</A>>
+
+<!--#include virtual="/includes/footer" -->
+
+</BODY>
+</HTML>
Added: trunk/docs/flush/docs/access.html
===================================================================
--- trunk/docs/flush/docs/access.html 2005-07-28 14:50:56 UTC (rev 252)
+++ trunk/docs/flush/docs/access.html 2005-07-31 10:51:40 UTC (rev 253)
@@ -0,0 +1,42 @@
+<html>
+<head><title>Flush Documentation: Message Type Access Macros</title></head>
+
+<body bgcolor="#FFFFFF">
+
+<!--#include virtual="/includes/header-a" -->
+
+<a href="http://www.cnds.jhu.edu/research/group/flush_spread">
+<img src="flush_spread_title.gif" alt="FLUSH SPREAD" border=0>
+</a>
+
+<!--#include virtual="/includes/header-b" -->
+
+<h2><em>Message Type Access Macros</em></h2>
+
+Flush Spread uses the same message type access macros as Spread. In addition to these
+<a href="http://www.spread.org/docs/access.html">common macros</a>, Flush Spread also
+adds the following message type access macros:
+
+<pre>
+#define Is_flush_req_mess(serv_type)
+#define Is_subgroup_mess(serv_type)
+</pre>
+
+<ol>
+<li><tt>Is_flush_req_mess(serv_type):</tt></p>
+
+This macro checks the message type to determine if the message is a flush request message or not.
+Flush request messages are an important part of the view synchrony GCS semantics, see <a
+href="FL_flush.html">FL_flush</a> for an in-depth discussion of them.</p>
+
+<li><tt>Is_subgroup_mess(serv_type):</tt></p>
+
+This macro checks the message type to determine if the message is a subgroup-multicast or
+not. A subgroup-multicast is a message that is only sent to a subset of a group, see
+<a href="FL_multicast.html">FL_subgroupcast</a> for a better description.</p>
+
+</ol>
+
+<!--#include virtual="/includes/footer" -->
+
+</body></html>
Added: trunk/docs/flush/docs/error.html
===================================================================
--- trunk/docs/flush/docs/error.html 2005-07-28 14:50:56 UTC (rev 252)
+++ trunk/docs/flush/docs/error.html 2005-07-31 10:51:40 UTC (rev 253)
@@ -0,0 +1,80 @@
+<html>
+<head><title>Flush Documentation: Error Return Codes</title></head>
+
+<body bgcolor="#FFFFFF">
+
+<!--#include virtual="/includes/header-a" -->
+
+<a href="http://www.cnds.jhu.edu/research/group/flush_spread">
+<img src="flush_spread_title.gif" alt="FLUSH SPREAD" border=0>
+</a>
+
+<!--#include virtual="/includes/header-b" -->
+
+<h2><em>Error Return Codes</em></h2>
+
+Flush Spread uses the same error return codes as Spread. In addition to these
+<a href="http://www.spread.org/docs/error.html">common error codes</a>, Flush Spread
+also adds the following error codes:
+
+<pre>
+#define ILLEGAL_PARAM
+#define WOULD_BLOCK
+#define ILLEGAL_MESSAGE_TYPE
+#define ILLEGAL_STATE
+#define ILLEGAL_RECEIVERS
+</pre>
+
+<ol>
+<li><tt>ILLEGAL_PARAM:</tt></p>
+
+An illegal parameter was passed to a call. Usually this is the result of specifying a
+negative array size.</p>
+
+<li><tt>WOULD_BLOCK:</tt></p>
+
+A receive call that requested the <tt>DONT_BLOCK</tt> <a href="messages.html">service</a> would
+have blocked.</p>
+
+<li><tt>ILLEGAL_MESSAGE_TYPE:</tt></p>
+
+A multicast call was made with an <a href="misc.html">illegal message type</a>.</p>
+
+<li><tt>ILLEGAL_STATE:</tt></p>
+
+A <a href="FL_multicast.html">FL_multicast</a> or <a href="FL_flush.html">FL_flush</a> call was made
+on a group while that group was in a prohibited state for that call.</p>
+
+<li><tt>ILLEGAL_RECEIVERS:</tt></p>
+
+A subgroup-multicast or unicast call specified receivers that were not currently
+members of the reference group.</p>
+</ol>
+
+Flush Spread also expands the meaning of the following Spread error codes:
+
+<pre>
+#define ILLEGAL_GROUP
+#define ILLEGAL_MESSAGE
+</pre>
+
+<ol>
+<li><tt>ILLEGAL_GROUP:</tt></p>
+
+In addition to representing illegal Spread group names, this error also represents that a
+group-specific call (i.e. - multicast, flush, join, leave) was made illegally while either
+not a member of the group (multicast, flush, leave) or while a member of the group
+(join).</p>
+
+<li><tt>ILLEGAL_MESSAGE:</tt></p>
+
+This error code in Flush Spread has the same meaning as it does in Spread. It means that
+either a send or receive message buffer or scatter was illegal in some manner. This can be
+returned if the size of a buffer is negative or if the number of scat elements is illegal
+(negative or greater than <tt><a href="misc.html">FL_MAX_SCATTER_ELEMENTS</a></tt>).</p>
+
+</ol>
+
+<!--#include virtual="/includes/footer" -->
+
+</body></html>
Added: trunk/docs/flush/docs/flush_spread_title.gif
===================================================================
(Binary files differ)
Property changes on: trunk/docs/flush/docs/flush_spread_title.gif
___________________________________________________________________
Name: svn:mime-type
+ application/octet-stream
Added: trunk/docs/flush/docs/functions.html
===================================================================
--- trunk/docs/flush/docs/functions.html 2005-07-28 14:50:56 UTC (rev 252)
+++ trunk/docs/flush/docs/functions.html 2005-07-31 10:51:40 UTC (rev 253)
@@ -0,0 +1,38 @@
+<html>
+<head><title>Flush Documentation: Flush Spread C Function Interface</title></head>
+
+<body bgcolor="#FFFFFF">
+
+<!--#include virtual="/includes/header-a" -->
+
+<a href="http://www.cnds.jhu.edu/research/group/flush_spread">
+<img src="flush_spread_title.gif" alt="FLUSH SPREAD" border=0>
+</a>
+
+<!--#include virtual="/includes/header-b" -->
+
+<h2><em>Flush Spread C Function Interface</em></h2>
+
+<ul>
+<li><a href="FL_version.html">FL_version</a>
+<li><a href="FL_connect.html">FL_connect</a>
+<li><a href="FL_disconnect.html">FL_disconnect</a>
+<li><a href="FL_join.html">FL_join</a>
+<li><a href="FL_leave.html">FL_leave</a>
+<li><a href="FL_flush.html">FL_flush</a>
+<li><a href="FL_multicast.html">FL_unicast</a>
+<li><a href="FL_multicast.html">FL_scat_unicast</a>
+<li><a href="FL_multicast.html">FL_subgroupcast</a>
+<li><a href="FL_multicast.html">FL_scat_subgroupcast</a>
+<li><a href="FL_multicast.html">FL_multicast</a>
+<li><a href="FL_multicast.html">FL_scat_multicast</a>
+<li><a href="FL_receive.html">FL_receive</a>
+<li><a href="FL_receive.html">FL_scat_receive</a>
+<li><a href="FL_more_msgs.html">FL_more_msgs</a>
+<li><a href="FL_poll.html">FL_poll</a>
+<li><a href="FL_error.html">FL_error</a>
+</ul>
+
+<!--#include virtual="/includes/footer" -->
+
+</body></html>
\ No newline at end of file
Added: trunk/docs/flush/docs/index.html
===================================================================
--- trunk/docs/flush/docs/index.html 2005-07-28 14:50:56 UTC (rev 252)
+++ trunk/docs/flush/docs/index.html 2005-07-31 10:51:40 UTC (rev 253)
@@ -0,0 +1,115 @@
+<html>
+<head><title>Flush Documentation</title></head>
+
+<body bgcolor="#FFFFFF">
+
+<!--#include virtual="/includes/header-a" -->
+
+<a href="http://www.cnds.jhu.edu/research/group/flush_spread">
+<img src="flush_spread_title.gif" alt="FLUSH SPREAD" border=0>
+</a>
+
+<!--#include virtual="/includes/header-b" -->
+
+<h2>Documentation</h2>
+
+Flush Spread is an extension to the <a href="http://www.spread.org/">Spread Wide Area Group
+Communication System</a>. Flush Spread and Spread are extremely similiar group communication
+systems (GCSs), in the services that they provide, and in their general interface. Therefore,
+reading Spread's <a href="http://www.spread.org/docs/docspread.html">documentation</a> before
+reading Flush Spread's documentation is highly recommended to get a general grasp of group
+communication and to fill in any holes in this documentation :).</p>
+
+There are two main differences between Spread and Flush Spread:
+(1) Spread provides <a href="http://www.cs.jhu.edu/~yairamir/dcs-94.ps ">extended virtual
+synchrony</a> GCS semantics while Flush Spread provides view synchrony GCS semantics, and (2) Spread
+provides open-group semantics while Flush Spread only provides closed-group semantics.</p>
+
+<h3>Extended Virtual Synchrony (EVS) vs. View Synchrony Semantics (VS)</h3></p>
+
+The main difference between these two models is how they handle view/membership changes. In an EVS
+system, membership changes occur without a client's intervention, whereas in a VS system a client
+must give its permission before a new view/membership can be installed. Of course, in "real-life" a
+VS system cannot truly hold-off membership changes (as other clients can crash, network can
+partition, etc.), but what it can do is insulate a client from these changes and not allow new
+members (who either joined or merged) or their messages to be presented to the client without its
+permission.</p>
+
+In a VS system, when an underlying membership change occurs (another member joins/leaves/disconnects,
+network parititions or merges) the GCS generates a flush request message indicating that the current
+view is out-of-date and requesting permission to install a new view. The client is still allowed to
+send messages at this point, and they may be able to receive messages from surviving members. When
+the client is ready, it flushes the group, which gives the GCS permission to install a new view.
+After flushing, a client is not allowed to send any messages to the flushed group until they receive
+the new view of the group.</p>
+
+In a EVS system, when an underlying membership change occurs the GCS figures out the new membership
+and delivers a new view to the client in its normal stream of messages.</p>
+
+In both systems, using the safety guarantees of the different message types and the virtual synchrony
+property (i.e. transitional sets) the client can infer some information about what other members have
+seen without further communication.</p>
+
+Although the difference in these models seems slight, it can lead to quite astonishing differences
+in performance and ease of programming. In general, the EVS model is faster and scales better with
+respect to membership changes, especially for simple group changes such as joins and leaves. But
+this performance comes at a price, namely that the client has less control over what is transpiring
+and therefore in general it is harder to program algorithms under EVS than under VS. For example,
+under EVS the sender doesn't know which other clients might receive its messages until the message
+is delivered back to the sender. Also, membership changes can come fast and furious and your
+algorithm must be tolerant to every combination of such changes. Under VS, the client knows exactly
+which other clients <b>might</b> receive its message when it sends -- so the application doesn't
+have to worry about some arbitrary client receiving its messages and mis-interpreting them, for
+example. Also, membership changes only occur as fast as the applications want them to, although
+several underlying memberships may be collapsed into one. So they are generally easier to handle
+and track.</p>
+
+<h3>Open-Group vs. Closed-Group Semantics</h3></p>
+
+Open-group semantics allows clients to perform group specific calls, such as multicasts, without
+being a member of the group in question. Closed-group semantics requires a client to be a full
+fledged member of a group (see below) in order to perform group specific calls. In general, a
+closed-group GCS is much more picky than an open-group system about a client's state and the
+operations they perform with respect to any particular group.</p>
+
+Under Spread, a client can call group specific calls such as join, leave, and *cast (multicast and
+its variants) pretty much whenever they want regardless of their membership in that group. Also,
+under Spread the guarantees of multicast messages span differerent groups. For example, if a sender
+sends two FIFO messages to two different groups and a receiver receives both messages, they will
+always receive the first and then the second FIFO message.</p>
+
+Under Flush Spread, a client can make group specific calls such as join, leave, flush, and *cast
+only when in the proper membership state with respect to that group. For example, leave, flush, and
+*cast calls can only be made when the client is a full-fledged member of the group. A full-fledged
+member is a client that has joined a group, received a membership including them in the group and is
+not in the process of leaving the group. Join calls can only be made when the client is not a
+member of the group (either never joined it or whose most recent membership for that group was a
+self-leave) and not already in the process of joining that group.</p>
+
+Also under Flush Spread, *cast and flush calls (i.e. FL_flush) are restricted depending on the group
+membership state. Flush calls can only be made <b>ONCE</b> per view/membership in response to
+receiving a flush request message. Once a client flushes a group they are not allowed to *cast to
+that group until they receive the next view/membership for that group. Flush Spread also does not
+support cross group message semantics. For example, if a sender sends two FIFO messages to two
+different groups and a receiver receives both messages, they may receive the first and then the
+second <b>OR</b> vice-versa.</p>
+
+<hr>
+
+<h2>C/C++ Documentation</h2>
+
+The following documentation explains the user level C interface to Flush Spread as specified in the
+header file <a href="../src/fl.h">fl.h</a>. This documentation assumes a knowledge of the ideas,
+discussed above, of view synchronous group communication.
+
+<ol>
+<li><a href="messages.html">Message Types for Data and Membership Messages</a>
+<li><a href="access.html">Message Type Access Functions</a>
+<li><a href="error.html">Error Return Codes</a>
+<li><a href="misc.html">Miscellaneous Constants</a>
+<li><a href="functions.html">Flush Spread C Function Interface</a>
+</ol>
+
+<!--#include virtual="/includes/footer" -->
+
+</body></html>
Added: trunk/docs/flush/docs/messages.html
===================================================================
--- trunk/docs/flush/docs/messages.html 2005-07-28 14:50:56 UTC (rev 252)
+++ trunk/docs/flush/docs/messages.html 2005-07-31 10:51:40 UTC (rev 253)
@@ -0,0 +1,61 @@
+<html>
+<head><title>Flush Documentation: Message Types for Data and Membership Messages</title></head>
+
+<body bgcolor="#FFFFFF">
+
+<!--#include virtual="/includes/header-a" -->
+
+<a href="http://www.cnds.jhu.edu/research/group/flush_spread">
+<img src="flush_spread_title.gif" alt="FLUSH SPREAD" border=0>
+</a>
+
+<!--#include virtual="/includes/header-b" -->
+
+<h2><em>Message Types for Data and Membership Messages</em></h2>
+
+Flush Spread uses the same service and membership types as Spread. In addition to these
+<a href="http://www.spread.org/docs/messages.html">common types</a>, Flush Spread also
+adds the following service types:
+
+<pre>
+#define DONT_BLOCK
+#define FLUSH_REQ_MESS
+#define SUBGROUP_CAST
+</pre>
+
+<ol>
+
+<li><tt>DONT_BLOCK:</tt></p>
+
+This service type can be requested when making a receive call. If the receive call would block while
+receiving on the mailbox, the call will fail and return the error code <tt>WOULD_BLOCK</tt>.
+Currently, Spread client libraries (v3.14 and earlier) do not support non-blocking receives. This
+means that even if the caller sets the <tt>DONT_BLOCK</tt> service flag a call to receive may take
+longer than would normally be expected of a non-blocking I/O call. However, the call should never
+block permanently.</p>
+
+<li><tt>FLUSH_REQ_MESS:</tt></p>
+
+This message type can be set upon returning from a receive call. It indicates that the received
+message was a flush request message for the group contained in sender. A flush request message is
+generated by Flush Spread when the underlying membership of that group changes. To install the new
+membership and make progress, the receiver must eventually respond to this signal by flushing that
+group by calling <a href="FL_flush.html">FL_flush</a>. Receiving a flush request message is an
+important part of the view synchrony GCS semantics and has pretty drastic importance. See <a
+href="FL_receive.html">FL_receive</a> for a full description of the restrictions incurred by and
+ramifications of receiving a flush request. Note that a flush request is neither a membership message
+nor a regular message. See the <a href="access.html">message type access functions</a> for a macro to
+test if a message is a flush request message.</p>
+
+<li><tt>SUBGROUP_CAST:</tt></p>
+
+This message type can be set upon returning from a receive call. It indicates that the
+received message was only multicasted to a subset of a group. This message was only sent
+to the members in the groups array in the range [0, *num_groups-1). The group that this
+subgroup-multicast occurred in is contained in groups[*num_groups-1].</p>
+
+</ol>
+
+<!--#include virtual="/includes/footer" -->
+
+</body></html>
Added: trunk/docs/flush/docs/misc.html
===================================================================
--- trunk/docs/flush/docs/misc.html 2005-07-28 14:50:56 UTC (rev 252)
+++ trunk/docs/flush/docs/misc.html 2005-07-31 10:51:40 UTC (rev 253)
@@ -0,0 +1,33 @@
+<html>
+<head><title>Flush Documentation: Miscellaneous Constants</title></head>
+
+<body bgcolor="#FFFFFF">
+
+<!--#include virtual="/includes/header-a" -->
+
+<a href="http://www.cnds.jhu.edu/research/group/flush_spread">
+<img src="flush_spread_title.gif" alt="FLUSH SPREAD" border=0>
+</a>
+
+<!--#include virtual="/includes/header-b" -->
+
+<h2><em>Miscellaneous Constants</em></h2>
+
+<pre>
+#define FL_MAX_SCATTER_ELEMENTS
+</pre>
+
+Flush Spread restricts the maximum number of scatter elements a caller is allowed to use
+in multicast calls to be less than or equal to <tt>FL_MAX_SCATTER_ELEMENTS</tt>. See Spread's
+<tt><a href="http://www.spread.org/docs/types.html">MAX_SCATTER_ELEMENTS</a></tt>.</p>
+
+<pre>
+#define FL_MIN_LEGAL_MESS_TYPE
+</pre>
+
+Flush Spread restricts the range of message types (int16) a caller is allowed to use in
+multicast calls to be greater than or equal to <tt>FL_MIN_LEGAL_MESS_TYPE</tt>.</p>
+
+<!--#include virtual="/includes/footer" -->
+
+</body></html>
Added: trunk/docs/flush/index.html
===================================================================
--- trunk/docs/flush/index.html 2005-07-28 14:50:56 UTC (rev 252)
+++ trunk/docs/flush/index.html 2005-07-31 10:51:40 UTC (rev 253)
@@ -0,0 +1,56 @@
+<html>
+<head><title>The Group Communication Project</title></head>
+
+<body bgcolor="#FFFFFF">
+
+<!--#include virtual="/includes/header-a" -->
+
+<img src="docs/flush_spread_title.gif" alt="FLUSH SPREAD">
+
+<!--#include virtual="/includes/header-b" -->
+
+</p>
+<font size= -1>
+<DD><A href="http://www.cs.jhu.edu/~yairamir">Yair Amir</A></DD>
+<DD><A href="http://www.cnds.jhu.edu/~jschultz">John Schultz</A></DD>
+<DD><A href="http://www.cnds.jhu.edu/~jonathan">Jonathan Stanton</A></DD>
+</font>
+<HR>
+</p>
+
+<p>
+The Flush Spread project focuses on providing view synchrony group communication
+semantics built on top of Spread's extended virtual synchrony semantics. Currently, these
+semantics are provided by a client library that links with Spread's client library.
+</p>
+
+<p>
+Obviously you need Spread for Flush Spread to work ... so go check out
+<a href="http://www.spread.org">Spread</a>.
+</p>
+
+<p>
+Flush Spread includes a distribution of the Stdutil library, which is a toolkit of
+high-performance C data structures and utility functions. If you are interested, you
+can learn more about it <a href="http://www.cnds.jhu.edu/software/stdutil">here</a>.
+</p>
+
+<H3>Flush Spread Resources:</H3>
+
+<UL>
+<li>Read Flush Spread's <a href="README">README</a> file.
+
+<li>Read the Flush Spread <a href="FLUSH_LICENSE">license</a> or the Stdutil
+ <a href="stdutil/STDUTIL_LICENSE">license</a>.<br>
+
+<li>Read the Flush Spread <a href="docs/index.html">documentation</a>.<br>
+
+<li><a href="http://www.cnds.jhu.edu/download/flush_spread.tar.gz">Download</a> Flush Spread now!<br>
+</UL>
+
+E-mail <a href="mailto:flush_spread at cnds.jhu.edu"><i>flush_spread at cnds.jhu.edu</i></a>
+for more information.</p>
+
+<!--#include virtual="/includes/footer" -->
+
+</body></html>
Added: trunk/docs/flush/man/FL_connect.3
===================================================================
--- trunk/docs/flush/man/FL_connect.3 2005-07-28 14:50:56 UTC (rev 252)
+++ trunk/docs/flush/man/FL_connect.3 2005-07-31 10:51:40 UTC (rev 253)
@@ -0,0 +1,105 @@
+.TH FL_connect 3 "Dec 2000" "Flush Spread" "User Manuals"
+.SH NAME
+FL_connect \- connect an application to a Spread daemon using Flush Spread semantics.
+.SH SYNOPSIS
+.B #include <fl.h>
+
+.BI "int FL_connect(const char *" daemon_name ", const char *" user_name ", int " priority ", int " group_membership ", mailbox *" mbox ", char *" private_name );
+.SH DESCRIPTION
+.B FL_connect
+is the initial call an application must make to establish a
+connection with a Spread daemon. All other Flush Spread calls
+refer to a valid
+.I mbox
+returned by this function.
+
+The
+.I daemon_name
+is the name of the Spread daemon with which to connect. It should be a
+string in one of the following forms:
+.RS
+.TP
+.B "4803"
+This will connect to the Spread daemon on the local
+machine running on port 4803. This form cannot be
+used to connect to a Windows95/NT machine.
+.TP
+.B "4803 at localhost"
+This will also connect to the Spread daemon
+on the local machine running on port 4803.
+This form can be used on Windows95/NT machines.
+.TP
+.BR "4803 at host.domain.edu " or " 4803 at 128.220.221.99"
+This will connect to the machine identified by either
+domain name or IP address at the specified port.
+.RE
+
+The
+.I user_name
+is the name this connection would like to be known as. It must be
+unique on the machine running the spread daemon. The name can be an
+arbitrary length string with the same character restrictions as a
+group name (mainly it cannot contain the '#' character).
+
+.I priority
+is a boolean (non-zero true, zero false) for whether this connection
+will be a "priority" connection or not. Currently this has no effect.
+
+.I group_membership
+is a boolean for whether this connection will receive group membership
+messages or not. Usually for Flush Spread applications this parameter
+should be true. If your application doesn't need group membership
+messages, then Spread may provide the semantics that you need more
+efficiently than Flush Spread.
+
+The
+.I mbox
+should be a pointer to a mailbox variable. After the
+.B FL_connect
+call successfully returns, this variable will hold the valid mbox for
+this new connection.
+
+The
+.I private_name
+should be a pointer to a string big enough to hold at least
+MAX_GROUP_NAME characters. After the call returns it will contain the
+private group name of this connection. These group names are what are
+reported in membership messages and can be used to send unicast and
+subgroup-multicast messages to this connection. No other applications
+can join this special group.
+
+.SH "RETURN VALUES"
+Returns
+.B ACCEPT_SESSION
+on success or one of the following errors ( < 0 ):
+.TP
+.B ILLEGAL_SPREAD
+The
+.I daemon_name
+given to connect to was illegal for some reason. Usually because
+it was a unix socket on Windows95/NT, an improper format for a host
+or an illegal port number
+.TP
+.B COULD_NOT_CONNECT
+Lower level socket calls failed to allow a connection to the
+specified spread daemon right now.
+.TP
+.B CONNECTION_CLOSED
+During communication to establish the connection errors occured
+and the setup could not be completed.
+.TP
+.B REJECT_VERSION
+The daemon and/or libraries have a version mismatch.
+.TP
+.B REJECT_NO_NAME
+No user name was provided.
+.TP
+.B REJECT_ILLEGAL_NAME
+Name provided violated some requirement (length or used an illegal character)
+.TP
+.B REJECT_NOT_UNIQUE
+Name provided is not unique on this daemon. Recommended response is to try
+again with a different name.
+.SH AUTHOR
+John Schultz <jschultz at cnds.jhu.edu>
+
Added: trunk/docs/flush/man/FL_disconnect.3
===================================================================
--- trunk/docs/flush/man/FL_disconnect.3 2005-07-28 14:50:56 UTC (rev 252)
+++ trunk/docs/flush/man/FL_disconnect.3 2005-07-31 10:51:40 UTC (rev 253)
@@ -0,0 +1,26 @@
+.TH FL_disconnect 3 "Dec 2000" "Flush Spread" "User Manuals"
+.SH NAME
+FL_disconnect \- destroy a Flush Spread connection between an application and a Spread daemon.
+.SH SYNOPSIS
+.B #include <fl.h>
+
+.BI "int FL_disconnect(mailbox " mbox ");"
+.SH DESCRIPTION
+.B FL_disconnect
+should be called when the application is finished
+with a connection to a Spread daemon. The application may have
+other connections still open to the daemon and may open a new
+connection after disconnecting. Any data that was available on
+the connection will be lost after the call returns.
+
+The
+.I mbox
+should be for the connection you wish to close.
+.SH "RETURN VALUES"
+Returns 0 on success or
+.B ILLEGAL_SESSION
+when the
+.I mbox
+given does not represent a valid connection.
+.SH AUTHOR
+John Schultz <jschultz at cnds.jhu.edu>
Added: trunk/docs/flush/man/FL_error.3
===================================================================
--- trunk/docs/flush/man/FL_error.3 2005-07-28 14:50:56 UTC (rev 252)
+++ trunk/docs/flush/man/FL_error.3 2005-07-31 10:51:40 UTC (rev 253)
@@ -0,0 +1,13 @@
+.TH FL_error 3 "Dec 2000" "Flush Spread" "User Manuals"
+.SH NAME
+FL_error \- Flush Spread error string reporting
+.SH SYNOPSIS
+.B #include <fl.h>
+
+.BI "void FL_error(int " error_code ");"
+.SH DESCRIPTION
+.B FL_error
+prints an error string based on the Flush Spread or Spread error code
+.IR error_code .
+.SH AUTHOR
+John Schultz <jschultz at cnds.jhu.edu>
Added: trunk/docs/flush/man/FL_flush.3
===================================================================
--- trunk/docs/flush/man/FL_flush.3 2005-07-28 14:50:56 UTC (rev 252)
+++ trunk/docs/flush/man/FL_flush.3 2005-07-31 10:51:40 UTC (rev 253)
@@ -0,0 +1,94 @@
+.TH FL_flush 3 "Dec 2000" "Flush Spread" "User Manuals"
+.SH NAME
+FL_flush \- flush a group in response to receiving a flush request message.
+.SH SYNOPSIS
+.B #include <fl.h>
+
+.BI "int FL_flush(mailbox " mbox ", const char *" group_name ");"
+.SH DESCRIPTION
+.B FL_flush
+flushes the group named
+.I group_name
+in which the connection represented by
+.I mbox
+has received a flush request message. A flush request message is
+delivered in a group when the underlying membership of that group
+changes; it is a signal to the application that the membership has
+changed and some of the original members may have "gone away" (meaning
+that they are no longer in the group with this connection). However,
+Flush Spread will not install a new membership in the group until each
+of the members of the new membership flush the group. Flushing a
+group gives Flush Spread the permission of this connection to go ahead
+and install a new view/membership for the group.
+
+During the time after receiving a flush request message but before
+flushing the group the application must be _VERY_ careful about
+calling receive functions; if the application isn't careful enough it
+can permantently block the entire group. For example, say the
+underlying membership of a group of 10 members changed such that this
+connection is now alone in the group (there is no way an application
+can detect the make-up of a group after receiving a flush request).
+If the application simply calls receive before flushing the group and
+hasn't ensured that it will receive a message somehow (see below) it
+will permanently block itself in the receive and any other members
+that might later be added to that group.
+
+There are a couple ways to avoid this problem, which isn't a bug by
+the way :P, (1) don't call receive functions after receiving a flush
+request message and before flushing the group, (2) call receive but
+set the DONT_BLOCK service flag; if there aren't any messages to
+receive it will break out with a WOULD_BLOCK error, (3) ensure that
+there will be a message to receive somehow. There are several ways to
+do this, but one 100% sure way to do this is if the application sends
+or has sent a message on this connection (with the SELF_DISCARD
+service _NOT_ set) that it has not yet received back. In this case,
+the application will eventually receive its own message (even if it
+was to another group) on the connection and can therefore be assured
+of not blocking permanently.
+
+The application _IS_ allowed to send messages to a group after
+receiving a flush request for that group and before flushing the
+group. However, only a subset of the original members of the current
+view will receive these messages (note that I said a subset, not a
+strict subset). Technically, this is always the case in Spread: an
+application can determine which other members received certain
+messages for sure by (1) application level message receipt
+acknowledgement, (2) employing the safety properties of SAFE messages
+(although this doesn't guarantee that those members actually processed
+and handled the message), (3) employing the virtual synchrony property
+and the transitional set of the new membership when it is installed
+(although, again, this doesn't guarantee that those members actually
+processed and handled the messages). See FL_receive or read up on
+group communication for more in-depth discussions of this matter.
+
+Anyways, when the application is ready it can flush the group. Once
+it flushes the group it is not allowed to send any messages to that
+group, until it receives the new membership for that group. If the
+application breaks this rule, it will receive an ILLEGAL_STATE error.
+
+.SH "RETURN VALUES"
+Returns 0 on success or one of the following errors ( < 0 ):
+.TP
+.B ILLEGAL_SESSION
+The connection represented by
+.I mbox
+is illegal. Usually because it is not active.
+.TP
+.B ILLEGAL_GROUP
+The
+.I group_name
+given to flush was illegal for some reason, usually because it was of
+length 0 or length > MAX_GROUP_NAME. This error can also be returned
+when a group is flushed for a connection that it has not yet joined or
+is already leaving.
+.TP
+.B ILLEGAL_STATE
+A connection may flush a group only once in response to each flush
+request message delivered in that group. If you violate this rule you
+will get an ILLEGAL_STATE error.
+.TP
+.B CONNECTION_CLOSED
+Errors occurred during communication and the flush could not be
+initiated.
+.SH AUTHOR
+John Schultz <jschultz at cnds.jhu.edu>
Added: trunk/docs/flush/man/FL_join.3
===================================================================
--- trunk/docs/flush/man/FL_join.3 2005-07-28 14:50:56 UTC (rev 252)
+++ trunk/docs/flush/man/FL_join.3 2005-07-31 10:51:40 UTC (rev 253)
@@ -0,0 +1,38 @@
+.TH FL_join 3 "Dec 2000" "Flush Spread" "User Manuals"
+.SH NAME
+FL_join \- join a connection to a Flush Spread group
+.SH SYNOPSIS
+.B #include <fl.h>
+
+.BI "int FL_join(mailbox " mbox ", const char *" group_name ");"
+.SH DESCRIPTION
+.B FL_join
+joins the connection represented by
+.I mbox
+to a group with the name
+.IR group_name .
+If the group does not exist among the Spread daemons it is created,
+otherwise the connection is joined to the existing group. A
+connection may not join a group it is already a member of, which it
+has already joined, or from which it is currently leaving.
+.SH "RETURN VALUES"
+Returns 0 on success or one of the following errors ( < 0 ):
+.TP
+.B ILLEGAL_GROUP
+The
+.I group_name
+given to join was illegal for some reason, usually because it was of
+length 0 or length > MAX_GROUP_NAME. This error is also returned if a
+group with which this connection is already involved (i.e. - already
+joining, already joined, currently leaving) is joined again.
+.TP
+.B ILLEGAL_SESSION
+The connection represented by
+.I mbox
+is illegal. Usually because it is not active.
+.TP
+.B CONNECTION_CLOSED
+Errors occurred during communication and the join could not be
+initiated.
+.SH AUTHOR
+John Schultz <jschultz at cnds.jhu.edu>
Added: trunk/docs/flush/man/FL_leave.3
===================================================================
--- trunk/docs/flush/man/FL_leave.3 2005-07-28 14:50:56 UTC (rev 252)
+++ trunk/docs/flush/man/FL_leave.3 2005-07-31 10:51:40 UTC (rev 253)
@@ -0,0 +1,40 @@
+.TH FL_leave 3 "Dec 2000" "Flush Spread" "User Manuals"
+.SH NAME
+FL_leave \- remove a connection from a Flush Spread group
+.SH SYNOPSIS
+.B #include <fl.h>
+
+.BI "int FL_leave(mailbox " mbox ", const char *" group_name ");"
+.SH DESCRIPTION
+.B FL_leave
+removes the connection represented by
+.I mbox
+from a Flush Spread group with the name
+.IR group_name .
+A connection may only leave a group after it has been installed as a
+member of the group by being included in a Flush Spread
+view/membership. Also a connection may not leave a group if it has
+already requested to leave that group and the self-leave message has
+not yet been received.
+.SH "RETURN VALUES"
+Returns 0 on success or one of the following errors ( < 0 ):
+.TP
+.B ILLEGAL_GROUP
+The
+.I group_name
+given to leave was illegal for some reason. Usually because it was of
+length 0 or length > MAX_GROUP_NAME. This error is also returned if
+the connection was not a full-fledged member (i.e. - not yet included
+in a Flush membership message for that group, or already leaving) of
+the group.
+.TP
+.B ILLEGAL_SESSION
+The connection represented by
+.I mbox
+is illegal. Usually because it is not active.
+.TP
+.B CONNECTION_CLOSED
+Errors occurred during communication
+and the leave could not be initiated.
+.SH AUTHOR
+John Schultz <jschultz at cnds.jhu.edu>
Added: trunk/docs/flush/man/FL_more_msgs.3
===================================================================
--- trunk/docs/flush/man/FL_more_msgs.3 2005-07-28 14:50:56 UTC (rev 252)
+++ trunk/docs/flush/man/FL_more_msgs.3 2005-07-31 10:51:40 UTC (rev 253)
@@ -0,0 +1,41 @@
+.TH FL_more_msgs 3 "Dec 2000" "Flush Spread" "User Manuals"
+.SH NAME
+FL_more_msgs \- returns the number of complete messages buffered and
+ready to be received on a connection.
+.SH SYNOPSIS
+.B #include <fl.h>
+
+.BI "int FL_more_msgs(mailbox " mbox ");"
+.SH DESCRIPTION
+.B FL_more_msgs
+allows an application to check to see if any complete messages are
+already buffered and ready to be received on the connection represented by
+.IR mbox .
+
+.BR NOTE ,
+that this function
+.B CANNOT
+be used as an I/O polling function to check if a receive call should
+be done. If this function returns zero there still might be a message
+ready to receive on the connection: FL_poll, and file descriptor
+selects/polls can detect if there is activity (not necessarily a
+message) on a connection, while DONT_BLOCK receive semantics can
+detect whether or not there is a message on the connection and
+FL_more_msgs can only answer if there are any buffered messages
+already on the connection.
+
+This function is merely a helper function to re-get the current status
+of the
+.I more_messes
+parameter returned from a call to
+.BR FL_receive .
+.SH "RETURN VALUES"
+Returns the number of complete buffered messages ready to be received,
+or one of the following errors ( < 0):
+.TP
+.B ILLEGAL_SESSION
+The connection represented by
+.I mbox
+is illegal, usually because it is not active.
+.SH AUTHOR
+John Schultz <jschultz at cnds.jhu.edu>
Added: trunk/docs/flush/man/FL_multicast.3
===================================================================
--- trunk/docs/flush/man/FL_multicast.3 2005-07-28 14:50:56 UTC (rev 252)
+++ trunk/docs/flush/man/FL_multicast.3 2005-07-31 10:51:40 UTC (rev 253)
@@ -0,0 +1,162 @@
+.TH FL_multicast 3 "Dec 2000" "Flush Spread" "User Manuals"
+.SH NAME
+FL_unicast, FL_scat_unicast, FL_subgroupcast, FL_scat_subgroupcast, FL_multicast, FL_scat_multicast \- Multicast messages to subsets of Flush Spread groups.
+.SH SYNOPSIS
+.B #include <fl.h>
+
+.BI "int FL_unicast(mailbox " mbox ", service " serv_type ", const char *" group_name ", const char *" recvr_name ", int16 " mess_type ", int " mess_len ", const char *" mess ");"
+
+.BI "int FL_scat_unicast(mailbox " mbox ", service " serv_type ", const char *" group_name ", const char *" recvr_name ", int16 " mess_type ", const scatter *" scat ");"
+
+.BI "int FL_subgroupcast(mailbox " mbox ", service " serv_type ", const char *" group_name ", int " num_recvrs ", const char " recvr_names "[][MAX_GROUP_NAME], int16 " mess_type ", int " mess_len ", const char *" mess ");"
+
+.BI "int FL_scat_subgroupcast(mailbox " mbox ", service " serv_type ", const char *" group_name ", int " num_recvrs ", const char " recvr_names "[][MAX_GROUP_NAME], int16 " mess_type ", const char *" scat ");"
+
+.BI "int FL_unicast(mailbox " mbox ", service " serv_type ", const char *" group_name ", int16 " mess_type ", int " mess_len ", const char *" mess ");"
+
+.BI "int FL_scat_unicast(mailbox " mbox ", service " serv_type ", const char *" group_name ", int16 " mess_type ", const scatter *" scat ");"
+.SH DESCRIPTION
+.B FL_multicast
+and its variants all multicast a message from the connection represented by
+.I mbox
+to a subset of the membership of the Flush Spread group named
+.IR group_name .
+
+Note, that unlike Spread, Flush Spread does not provide guarantees
+about messages sent to different groups; meaning that FIFO, CAUSAL,
+SAFE, etc. safety properties are not maintained across different Flush
+Spread groups, but only within a group to which they are sent. For
+example, if an application sends a FIFO message A to the private group
+G and then sends another FIFO message B to the regular group H, of
+which G is a member, then G could receive B before A or vice versa.
+
+In order to multicast a message to a group, the destination group
+.I group_name
+must either be a private group name, or this connection must be a
+full-fledged member of that regular group (has joined and been
+included in a membership of the group, and is not leaving the group,
+see FL_join, FL_leave) and not be in an illegal state to multicast (by
+having recently flushed the group and not yet received its next
+view/membership, see FL_flush).
+
+The
+.I serv_type
+is a type field that should be set to the service type this message
+requires. The valid flags for multicasting messages are:
+
+.RS
+.B UNRELIABLE_MESS
+.br
+.TB
+.B RELIABLE_MESS
+.br
+.TB
+.B FIFO_MESS
+.br
+.TB
+.B CAUSAL_MESS
+.br
+.TB
+.B AGREED_MESS
+.br
+.TB
+.B SAFE_MESS
+.RE
+
+This type field can be bit-wise ORed with other flags like
+SELF_DISCARD if desired. Currently SELF_DISCARD is the only
+additional flag for multicasts: SELF_DISCARD will cause the
+multicasted message to NOT be delivered back to this connection.
+
+The string
+.I group_name
+indicates the name of the Flush Spread group to which this message
+should be sent. This group can be either a regular or a private Flush
+Spread group, as dicussed above. The multicast variants sends the
+message to all members of a group, while the unicast and subgroupcast
+variants allow the sender to specify a subset of the group to which to
+send the message. For the unicast variants the sender specifies the
+private group name of the intended recipient in the variable
+.IR recvr_name .
+For the subgroupcast variants the sender specifies the number of
+recipients in the variable
+.I num_recvrs
+and a doubly scripted array of the members' private group names in the
+variable
+.IR recvr_names ,
+where duplicate names are tolerated. For the unicast and subgroupcast
+variants all specified receivers must have been members of the most
+recently installed view/membership of the group. If the application
+breaks this rule it will receive an ILLEGAL_RECEIVERS error.
+
+The
+.I mess_type
+is a 16 bit integer which can be used by the application arbitrarily.
+However, Flush Spread restricts which values can be used to be greater
+than or equal to FL_MIN_LEGAL_MESS_TYPE. If the application sends
+with a message type of less than FL_MIN_LEGAL_MESS_TYPE, then the
+error ILLEGAL_MESSAGE_TYPE will be returned. The intent of this
+message type is that the application can use it to name different
+kinds of data messages so they can be differentiated without looking
+into the body of the message. This value
+.B WILL
+be endian corrected upon returning.
+
+The non-scatter variants use a single buffer to pass the body of the
+message to be sent. The
+.I mess_len
+field gives the message length in bytes. While the
+.I mess
+field is a pointer to the buffer containing the message. For the
+scatter variants, both of these parameters are replaced with one
+pointer,
+.IR scat_mess ,
+to a scatter structure, which is similiar to an iovec. This allows
+messages made up of several parts to be sent without an extra copy on
+systems which support scatter-gather. However, the number of scatter
+elements is restricted to be non-negative and less than or equal to
+FL_MAX_SCATTER_ELEMENTS. If a buffer length is negative or an illegal
+number of scatter elements is used then ILLEGAL_MESSAGE will be
+returned.
+.SH "RETURN VALUES"
+Returns the number of bytes sent on success or one of the following
+errors ( < 0 ):
+.TP
+.B ILLEGAL_SESSION
+The connection represented by
+.I mbox
+was illegal, usually because it is no longer active.
+.TP
+.B ILLEGAL_GROUP
+The
+.I group_name
+given to multicast to was illegal for some reason. Usually because it
+was of length 0 or length > MAX_GROUP_NAME. This error can also be
+returned if the connection was not a full-fledged member (i.e. - not
+yet included in a Flush membership message for the group, or already
+leaving) of a regular group.
+.TP
+.B ILLEGAL_RECEIVERS
+A unicast or subgroupcast specified private group names that weren't
+members of
+.IR group_name 's
+most recent view/membership.
+.TP
+.B ILLEGAL_STATE
+A multicast to a group was attempted after flushing the group and
+before receiving the next view/membership for that group.
+.TP
+.B ILLEGAL_SERVICE
+If the service type was using bits reserved by Flush Spread or Spread.
+.TP
+.B ILLEGAL_PARAM
+An illegal parameter, such as a negative array size was passed.
+.TP
+.B ILLEGAL_MESSAGE
+The message had an illegal structure, like a negative buffer size or
+an illegal number of scatter elements.
+.TP
+.B CONNECTION_CLOSED
+Errors occurred during communication and the multicast could not be completed.
+.SH AUTHOR
+John Schultz <jschultz at cnds.jhu.edu>
Added: trunk/docs/flush/man/FL_poll.3
===================================================================
--- trunk/docs/flush/man/FL_poll.3 2005-07-28 14:50:56 UTC (rev 252)
+++ trunk/docs/flush/man/FL_poll.3 2005-07-31 10:51:40 UTC (rev 253)
@@ -0,0 +1,32 @@
+.TH FL_poll 3 "Dec 2000" "Flush Spread" "User Manuals"
+.SH NAME
+FL_poll \- returns the amount, in bytes, of activity on a connection.
+.SH SYNOPSIS
+.B #include <fl.h>
+
+.BI "int FL_poll(mailbox " mbox ");"
+.SH DESCRIPTION
+.B FL_poll
+is a way to poll the connection represented by
+.I mbox
+to see if there is any activity on that connection.
+
+.BR NOTE ,
+however that activity does
+.B NOT
+necessarily mean that a message is available to be read, as in Spread.
+Only a call to FL_more_msgs that returns a positive number of buffered
+messages or a DONT_BLOCK receive call can check for sure to see if a
+message is now available to be read. If you don't use either of these
+semantics, then you may enter into a blocking receive call, which can
+sometimes be dangerous (see FL_flush).
+.SH "RETURN VALUES"
+Returns the number of bytes of activity currently on the connection,
+or one of the following errors ( < 0 ):
+.TP
+.B ILLEGAL_SESSION
+The connection represented by
+.I mbox
+is illegal, usually because it is not active.
+.SH AUTHOR
+John Schultz <jschultz at cnds.jhu.edu>
Added: trunk/docs/flush/man/FL_receive.3
===================================================================
--- trunk/docs/flush/man/FL_receive.3 2005-07-28 14:50:56 UTC (rev 252)
+++ trunk/docs/flush/man/FL_receive.3 2005-07-31 10:51:40 UTC (rev 253)
@@ -0,0 +1,388 @@
+.TH FL_receive 3 "Dec 2000" "Flush Spread" "User Manuals"
+.SH NAME
+Fl_receive, FL_scat_receive \- receive a message on a Flush Spread connection.
+.SH SYNOPSIS
+.B #include <fl.h>
+
+.BI "int FL_receive(mailbox " mbox ", service *" serv_type ", char " sender "[MAX_GROUP_NAME], int " max_groups ", int *" num_groups ", char " groups "[][MAX_GROUP_NAME], int16 *" mess_type ", int *" endian_mismatch ", int " max_mess_len ", char *" mess ", int *" more_messes ");"
+
+.BI "int FL_scat_receive(mailbox " mbox ", service *" serv_type ", char " sender "[MAX_GROUP_NAME], int " max_groups ", int *" num_groups ", char " groups "[][MAX_GROUP_NAME], int16 *" mess_type ", int *" endian_mismatch ", scatter *" scat_mess ", int *" more_messes ");"
+.SH DESCRIPTION
+.B FL_receive
+is the general purpose message receipt function for Flush Spread.
+Messages for all groups joined on this connection will arrive on the
+same mailbox. A call to
+.B FL_receive
+will perform a receive on a connection to get a single message from
+any one of the groups to which it is joined.
+
+After a call to receive completes, a number of the passed fields are
+set to values indicating meta-information about the message (such as
+destination group, message type, endianness, etc). The meanings of
+these different meta fields depends on the type of message received
+and the receipt services requested.
+
+Input Parameters:
+.RS
+.TP 1.1i
+.I mbox
+Represents the connection on which to receive a message.
+.TP 1.1i
+.I serv_type
+A pointer to a service bit field, requesting some receive services.
+This field can have the DONT_BLOCK and/or DROP_RECV service bit flags
+flipped on. This bit field should be zeroed out before each call if
+no special receive service is to be requested.
+.TP 1.1i
+.I sender
+A pointer to a character array with storage for at least
+MAX_GROUP_NAME characters.
+.TP 1.1i
+.I max_groups
+A non-negative int representing how many group names the application
+is willing to receive in this receive call.
+.TP 1.1i
+.I num_groups
+A pointer to an int.
+.TP 1.1i
+.I groups
+An array of group names containing storage for at least
+.I max_groups
+* MAX_GROUP_NAME characters.
+.TP 1.1i
+.I mess_type
+A pointer to an int16.
+.TP 1.1i
+.I endian_mismatch
+A pointer to an int.
+.TP 1.1i
+.I max_mess_len
+A non-negative integer representing how many bytes of data the
+application is willing to receive in this call.
+.TP 1.1i
+.I mess
+A pointer to a buffer containing storage for at least
+.I max_mess_len
+bytes.
+.TP 1.1i
+.I scat_mess
+A pointer to a scatter (a cousin of an iovec) with a non-negative
+number of scatter elements that is less than or equal to
+MAX_SCATTER_ELEMENTS (that's not a typo :), and all non-negative
+buffer lengths in the indicated scatter elements to get scatter-gather
+semantics.
+.RE
+
+Receive Semantics:
+
+Normally, a call to receive will block if no messages are immediately
+available.
+
+Normally, when calling a receive function if a user's buffer
+.RI ( groups ,
+.IR mess ,
+or
+.IR scat_mess )
+is too small to contain the data to be returned, then a
+GROUPS_TOO_SHORT or BUFFER_TOO_SHORT error code will be returned. In
+this case, the
+.IR serv_type ,
+and
+.IR mess_type
+of the message are filled in from the offending message and the
+parameters
+.IR num_groups ,
+and
+.I endian_mismatch
+reflect information about the necessary buffers' sizes. If
+.I max_groups
+was big enough, then
+.RI * num_groups
+will be zero, otherwise it will be the negative of how many group
+names are available to be received (i.e. -7 means the
+.I max_groups
+was less than 7 and 7 group names can be received). If
+.I max_mess_len
+or
+.I scat_mess
+was big enough then
+.RI * endian_mismatch
+will be zero, otherwise it will be the negative of how much data is
+available to be received (i.e. -32768 means the msg buffer was too
+small and 32768 bytes of data can be received). The offending message
+is still available to be received through later calls to receive with
+appropriately sized user buffers.
+
+The parameter
+.I serv_type
+allows the applications to request different receive services that
+affect the normal semantics of receive. Currently, the only services
+are DONT_BLOCK and DROP_RECV.
+
+The DONT_BLOCK service makes a receive call non-blocking. With this
+service, the receive call will return quickly either with a message or
+with the error code WOULD_BLOCK if no message is available. Using
+this service is the only way to detect if a message is ready on the
+connection in a non-blocking manner in Flush Spread.
+
+The DROP_RECV service forces Flush Spread to read the current message
+off of the connection regardless of whether or not the user's buffers
+are big enough to fit all of the data. In this case, a TOO_SHORT
+error code is still returned even though the call actually
+succeeded. Anyway, as much data as can be fit into the user's buffers
+will be stuffed into them, and that message will no longer be on the
+connection. Also,
+.RI * num_groups
+will still be set to the negative size of how many names were
+available (if
+.I groups
+wasn't big enough), but
+.RI * endian_mismatch
+will not reflect the size of the data that could have been received (if
+.I mess
+or
+.I scat_mess
+wasn't big enough), instead it indicates whether or not the sending
+machine had an opposite endianness or not. Using DROP_RECV in this
+manner, when a buffer problem occurs there is no way to determine how
+big the data portion of the message actually was. This service is
+meant to be used when a call to receive without DROP_RECV fails with a
+buffer error, but it is determined that the message isn't all that
+important, or something along those lines; otherwise reallocate your
+receive buffers appropriately and re-call receive again without
+the DROP_RECV service.
+
+Output Parameters:
+
+Upon a successful return from receive,
+.RI * serv_type
+will be filled out with the message type that was just received. The
+specific type of message received can be tested using the different
+message and membership type access macros. The rest of the
+parameters' meanings differ depending on the
+.IR serv_type .
+
+Regular Messages:
+
+If the
+.I serv_type
+is a REG_MESSAGE (i.e. a data message) then:
+
+The parameter
+.I sender
+will be filled with the private group name of the sending connection.
+
+The parameter
+.I num_groups
+will be set to the number of group names filled into
+.IR groups .
+
+The
+.I groups
+array will contain the group to which this message was sent. If the
+message is a SUBGROUP_CAST (check the service type, see FL_multicast)
+then all of the recipients' private group names will be listed and the
+last name in the array (i.e.
+.RI groups[* num_groups -1])
+will be the regular group to which this message was sent.
+
+If DROP_RECV is being used, and
+.I groups
+is too small, then as many names as can fit will be filled in as above
+described. If the message was a SUBGROUP_CAST, then the last group in
+the array (i.e.
+.RI groups[ max_groups -1])
+will be the destination group for the message. If
+.I max_groups
+is zero then even the destination group is not reported.
+
+The parameter
+.RI * mess_type
+will be set to the message type field the application sent with
+the original message, which is a restricted (see FL_multicast) 16 bit
+integer. This value is endian corrected upon returning.
+
+The parameter
+.RI * endian_mismatch
+will be set to true (non-zero) if the endianness of the sending
+machine is the opposite of this receiving machine's.
+
+The actual message body being received will be filled into either the
+.I mess
+or
+.IR scat_mess
+parameter.
+
+Flush Request Message:
+
+If this is a FLUSH_REQ_MESS then this represents that the underlying
+membership has changed and that this application needs to flush this
+group before the new view/membership can be installed (see
+FL_flush).
+
+The
+.I sender
+will be filled in with the name of the group this connection needs
+to flush. All the other fields are set to empty values.
+
+Membership Messages:
+
+If this is a MEMB_MESSAGE (i.e. membership message) and it
+specifically is a REG_MEMB_MESS, then:
+
+The
+.I sender
+will be filled with the name of the group for which the membership
+change is occuring.
+
+The
+.I mess_type
+field will be set to the index of this process in the
+.I groups
+array (see below).
+
+The
+.I endian_mismatch
+field will be set to 0 since there are no endian issues with regular memberships.
+
+The
+.I groups
+array and message
+body are used to provide two kinds of membership information about the change that just
+occured in this group. The
+.I num_groups
+field will be set to the number of members in the group in the
+.B new
+membership (i.e. after the change occured). Correspondingly, the
+.I groups
+array will be filled in with the private group names of all members of this
+group in the
+.B new
+membership. This list of names is always in the same order at all receipients
+and thus can be used to deterministically pick a group representative if
+one is needed by the application.
+
+The second set of information is stored in the message body. The data
+buffer will include the following fields:
+.RS
+.TP
+.B group_id vid;
+.br
+.TP
+.B int num_members;
+.br
+.TP
+.B char dgroups[][MAX_GROUP_NAME];
+.RE
+
+The vid and num_members parameters will already be endian corrected.
+The dgroups array will contain num_members group names. The content
+of the dgroups array is dependent upon the type of the membership
+change:
+.RS
+.TP 0.8i
+.B CAUSED_BY_JOIN:
+dgroups contains the private group of the joining process.
+.TP
+.B CAUSED_BY_LEAVE:
+dgroups contains the private group of the leaving process.
+.TP
+.B CAUSED_BY_DISCONNECT:
+dgroups contains the private group of the disconnecting process.
+.TP
+.B CAUSED_BY_NETWORK:
+dgroups contains the group names of the members of the new membership who came
+with
+.B this
+application to the new view/membership. Using this data the previous
+membership listing and the new membership listing an application can
+determine all the members that left and were added in the NETWORK
+membership change. Note, that a member can both leave and be added in
+a
+.B single
+NETWORK change.
+.RE
+
+Transitional Message:
+
+If this is a MEMB_MESSAGE and specifically it is a TRANSITION_MESS,
+then this means that one or more of the current members of the group
+has left and not all of the safety guarantees can be met w.r.t. all of
+the original members for the messages delivered after this signal and
+before the next view/membership. Transitional messages never have a
+CAUSED_BY type because they only serve to signal up to where SAFE
+delivery and AGREED delivery (with no holes) is guaranteed in the
+view/membership in which they are delivered. Only one TRANSITION_MESS
+is delivered per group view/membership per connection.
+
+The
+.I sender
+will be filled in with the name of the group for which the membership
+change is occurring. All the other parameters will be set to empty values.
+
+Self-Leave Message:
+
+If this is a MEMB_MESSAGE (i.e. membership message) and it is neither
+a REG_MEMB_MESS or a TRANSITION_MESS, then it represents exactly the
+situtation where the member receiving this message has left a group
+and this is notification that the leave has occured, thus it is
+sometimes called a self-leave message. The simplest test for this is
+if a message is CAUSED_BY_LEAVE and REG_MEMB_MESS is false (zero),
+then it is a self-leave message.
+
+The other members of the group that this member just left will receive
+a normal TRANSITION_MESS, REG_MEMB_MESS pair as described above
+showing this connection leaving.
+
+For self-leave messages the
+.I sender
+field will be filled in with the name of the group this connection is
+leaving. All the other fields are set to empty values.
+
+.SH "RETURN VALUES"
+Returns the size of the data portion of the message received on
+success or one of the following errors ( < 0 ):
+.TP 0.8i
+.B ILLEGAL_SESSION
+The connection represented by
+.I mbox
+was illegal, usually because it is not active.
+.TP
+.B ILLEGAL_PARAM
+An illegal parameter was passed, like a negative array size.
+.TP
+.B ILLEGAL_MESSAGE
+The msg buffer had an illegal structure, like an illegal number of
+scatter elements or a negatively sized buffer.
+.TP
+.B WOULD_BLOCK
+The receive call was made with the DONT_BLOCK service and it would
+have blocked because no messages were ready to be received.
+.TP
+.B GROUPS_TOO_SHORT
+The receive call was made with a groups array that was too small.
+.RI * num_groups
+is filled in with the negative number of names that could have been
+returned. Note, that the msg buffer could also be too short, and this
+should be determined by examining
+.I endian_mismatch
+as described above. If DROP_RECV was used this error code actually
+represents success (see above).
+.TP
+.B BUFFER_TOO_SHORT
+The receive call was made with a msg buffer that was too small.
+.RI * endian_mismatch
+is filled in with the negative number of bytes that could have been
+received if DROP_RECV isn't used. If it is used then this error code
+actually represents success and
+.I endian_mismatch
+has the normal meaning (see above). Note, that the groups buffer
+could also be too short, this should be determined by examining
+.I num_groups
+as described above.
+.TP
+.B CONNECTION_CLOSED
+During communication to receive, communication errors occurred
+and the receive could not be completed.
+.SH AUTHOR
+John Schultz <jschultz at cnds.jhu.edu>
Added: trunk/docs/flush/man/FL_scat_multicast.3
===================================================================
--- trunk/docs/flush/man/FL_scat_multicast.3 2005-07-28 14:50:56 UTC (rev 252)
+++ trunk/docs/flush/man/FL_scat_multicast.3 2005-07-31 10:51:40 UTC (rev 253)
@@ -0,0 +1 @@
+.so FL_multicast.3
Added: trunk/docs/flush/man/FL_scat_receive.3
===================================================================
--- trunk/docs/flush/man/FL_scat_receive.3 2005-07-28 14:50:56 UTC (rev 252)
+++ trunk/docs/flush/man/FL_scat_receive.3 2005-07-31 10:51:40 UTC (rev 253)
@@ -0,0 +1 @@
+.so FL_receive
Added: trunk/docs/flush/man/FL_scat_subgroupcast.3
===================================================================
--- trunk/docs/flush/man/FL_scat_subgroupcast.3 2005-07-28 14:50:56 UTC (rev 252)
+++ trunk/docs/flush/man/FL_scat_subgroupcast.3 2005-07-31 10:51:40 UTC (rev 253)
@@ -0,0 +1 @@
+.so FL_multicast.3
Added: trunk/docs/flush/man/FL_scat_unicast.3
===================================================================
--- trunk/docs/flush/man/FL_scat_unicast.3 2005-07-28 14:50:56 UTC (rev 252)
+++ trunk/docs/flush/man/FL_scat_unicast.3 2005-07-31 10:51:40 UTC (rev 253)
@@ -0,0 +1 @@
+.so FL_multicast.3
Added: trunk/docs/flush/man/FL_subgroupcast.3
===================================================================
--- trunk/docs/flush/man/FL_subgroupcast.3 2005-07-28 14:50:56 UTC (rev 252)
+++ trunk/docs/flush/man/FL_subgroupcast.3 2005-07-31 10:51:40 UTC (rev 253)
@@ -0,0 +1 @@
+.so FL_multicast.3
Added: trunk/docs/flush/man/FL_unicast.3
===================================================================
--- trunk/docs/flush/man/FL_unicast.3 2005-07-28 14:50:56 UTC (rev 252)
+++ trunk/docs/flush/man/FL_unicast.3 2005-07-31 10:51:40 UTC (rev 253)
@@ -0,0 +1 @@
+.so FL_multicast.3
Added: trunk/docs/flush/man/FL_version.3
===================================================================
--- trunk/docs/flush/man/FL_version.3 2005-07-28 14:50:56 UTC (rev 252)
+++ trunk/docs/flush/man/FL_version.3 2005-07-31 10:51:40 UTC (rev 253)
@@ -0,0 +1,14 @@
+.TH FL_version 3 "Dec 2000" "Flush Spread" "User Manuals"
+
+.SH NAME
+FL_version \- returns library version information.
+.SH SYNOPSIS
+.B #include <fl.h>
+
+.BI "void FL_version(int *" major_ver ", int *" minor_ver ", int *" patch_ver ");"
+.SH DESCRIPTION
+.B FL_version
+returns the major, minor, and patch version numbers of the Flush
+Spread library in use.
+.SH AUTHOR
+John Schultz <jschultz at cnds.jhu.edu>
Added: trunk/flush/Makefile.in
===================================================================
--- trunk/flush/Makefile.in 2005-07-28 14:50:56 UTC (rev 252)
+++ trunk/flush/Makefile.in 2005-07-31 10:51:40 UTC (rev 253)
@@ -0,0 +1,57 @@
+CC=@CC@
+AR=@AR@
+
+SP_INC_DIR = -I../include
+SP_LIB_DIR = -L../daemon
+
+STD_INC_DIR = -I../stdutil/src
+STD_LIB_DIR = -L../stdutil/lib
+
+# use thread safe libs and enable std locking mechanisms (optional)
+THREAD_SAFE = -D_REENTRANT
+
+# don't use debugging (asserts and dprintfs) (optional)
+NO_DEBUG = -DNDEBUG
+
+# libraries apps need to link with: pthreads (optional), flush, stdutil, spread
+LIBS = @LIBS@ $(TARGET_DIR)/libflush.a -ltspread -lstdutil
+
+# where to put the libraries (libflush.a, libstdutil.a, etc.) and exes
+TARGET_DIR = .
+APP_DIR = .
+
+# all the defined variables
+DEFINES = $(THREAD_SAFE) $(NO_DEBUG) $(DO_TIMINGS)
+
+# all the include directories
+INCLUDES = $(SP_INC_DIR) -I. -I../stdutil/src
+
+# all the library linkage diretories
+LDFLAGS = $(SP_LIB_DIR)
+
+# gcc specific compiler flags
+CFLAGS = -Wall -O $(INCLUDES) $(DEFINES) # -g
+
+TARGETS = $(TARGET_DIR)/libflush.a $(APP_DIR)/flush_user $(APP_DIR)/sp_time_memb $(APP_DIR)/fl_time_memb
+
+OBJS = fl.o scatp.o
+
+all: $(TARGETS)
+
+$(TARGET_DIR)/libflush.a: $(OBJS)
+ $(AR) -rs $(TARGET_DIR)/libflush.a $(OBJS)
+
+$(APP_DIR)/flush_user: $(TARGET_DIR)/libflush.a user.o
+ $(CC) $(CFLAGS) $(LDFLAGS) -o $(APP_DIR)/flush_user user.o $(LIBS)
+
+$(APP_DIR)/sp_time_memb: $(TARGET_DIR)/libflush.a sp_time_memb.o stats.o
+ $(CC) $(CFLAGS) $(LDFLAGS) -o $(APP_DIR)/sp_time_memb sp_time_memb.o stats.o $(LIBS)
+
+$(APP_DIR)/fl_time_memb: $(TARGET_DIR)/libflush.a fl_time_memb.o stats.o
+ $(CC) $(CFLAGS) $(LDFLAGS) -o $(APP_DIR)/fl_time_memb fl_time_memb.o stats.o $(LIBS)
+
+clean:
+ rm -f *~ *.o $(TARGETS) core *.bak
+
+distclean: clean
+ rm -f Makefile config.*
Added: trunk/flush/configure.in
===================================================================
--- trunk/flush/configure.in 2005-07-28 14:50:56 UTC (rev 252)
+++ trunk/flush/configure.in 2005-07-31 10:51:40 UTC (rev 253)
@@ -0,0 +1,8 @@
+AC_INIT(fl.c)
+
+AC_PROG_CC
+AC_CHECK_PROGS(AR, ar)
+AC_CHECK_LIB(pthread, pthread_mutex_init)
+AC_CHECK_LIB(m, sqrt)
+
+AC_OUTPUT(Makefile)
Added: trunk/flush/fl.c
===================================================================
--- trunk/flush/fl.c 2005-07-28 14:50:56 UTC (rev 252)
+++ trunk/flush/fl.c 2005-07-31 10:51:40 UTC (rev 253)
@@ -0,0 +1,2141 @@
+/*
+ * The contents of this file are subject to the FLUSH SPREAD Non-Commercial
+ * License, Version 1.0 (the ``License''); you may not use this file except
+ * in compliance with the License. You may obtain a copy of the License at:
+ *
+ * http://www.cnds.jhu.edu/research/group/flush_spread/FLUSH_LICENSE
+ *
+ * or in the file ``FLUSH_LICENSE'' found in the root of this distribution.
+ *
+ * Software distributed under the License is distributed on an AS IS basis,
+ * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+ * for the specific language governing rights and limitations under the
+ * License.
+ *
+ * The Original Software is:
+ * The Flush Spread Library
+ *
+ * The Initial Developers of the Original Software are:
+ * Yair Amir, John Schultz and Jonathan Stanton
+ *
+ * All Rights Reserved.
+ *
+ */
+
+#include <stdlib.h>
+
+#ifdef USE_DMALLOC
+#include <dmalloc.h>
+#endif
+
+#include <stdio.h>
+#include <stdarg.h>
+#include <string.h>
+#include <assert.h>
+#include <fl_p.h>
+#include <stdutil/stdutil.h>
+
+/* glob_conns is a look up table that maps mailboxes to fl_conn*s */
+static stdmutex glob_conns_lock;
+static stdhash glob_conns = STDHASH_STATIC_CONSTRUCT(sizeof(mailbox), sizeof(fl_conn*), NULL, NULL, 0);
+
+/********************************* public flush layer interface ********************************/
+
+int FL_lib_init(void)
+{
+ static stdbool first_time = STDTRUE; /* *TRY* to mitigate bad call race conditions to FL_lib_init */
+ int ret = ILLEGAL_STATE;
+
+ if (first_time) {
+ first_time = STDFALSE;
+ ret = stdmutex_construct(&glob_conns_lock, STDMUTEX_FAST);
+ }
+
+ return ret;
+}
+
+void FL_version(int *major, int *minor, int *patch) {
+ *major = FL_MAJOR_VERSION;
+ *minor = FL_MINOR_VERSION;
+ *patch = FL_PATCH_VERSION;
+}
+
+/* Establish a new fl connection. If SP_connect succceeds, create a new fl_conn instance. */
+int FL_connect(const char *daemon, const char *user, int priority,
+ mailbox *mbox, char private[MAX_GROUP_NAME]) {
+ int ret;
+ fl_conn *conn;
+
+ DEBUG(std_stkfprintf(stderr, 1, "FL_connect: daemon '%s', user '%s', priority %d\n",
+ daemon, user, priority));
+ if (FL_SP_version() < (float) 3.12) { /* flush depends on the DROP_RECV semantics */
+ DEBUG(std_stkfprintf(stderr, 0, "REJECT_VERSION: SP too old v%f < v3.12\n", FL_SP_version()));
+ ret = REJECT_VERSION;
+ } else if ((ret = SP_connect(daemon, user, priority, 1, mbox, private)) == ACCEPT_SESSION) {
+ DEBUG(std_stkfprintf(stderr, 0, "mbox %d, private '%s'\n", *mbox, private));
+
+ if ((conn = (fl_conn*) calloc(1, sizeof(fl_conn))) == 0)
+ stderr_abort("(%s, %d): calloc(1, %u)\n", __FILE__, __LINE__, sizeof(fl_conn));
+
+ stdmutex_construct(&conn->reserve_lock, STDMUTEX_FAST);
+ conn->reservations = 0;
+ conn->disconnecting = 0;
+ stdcond_construct(&conn->destroy_cond);
+
+ stdmutex_construct(&conn->recv_lock, STDMUTEX_FAST);
+ stdmutex_construct(&conn->conn_lock, STDMUTEX_FAST);
+
+ conn->mbox = *mbox;
+ conn->priority = priority;
+ conn->group_memb = 1;
+ strncpy(conn->daemon, daemon, MAX_GROUP_NAME);
+ strncpy(conn->user, user, MAX_GROUP_NAME);
+ strncpy(conn->private, private, MAX_GROUP_NAME);
+
+ stdhash_construct(&conn->groups, sizeof(char*), sizeof(fl_group*),
+ group_name_ptr_cmp, group_name_ptr_hashcode, 0);
+ stddll_construct(&conn->mess_queue, sizeof(gc_buff_mess*)); /* <gc_buff_mess*> */
+ conn->bytes_queued = 0;
+
+ stdmutex_grab(&glob_conns_lock); /* LOCK CONNS TAB */
+ stdhash_insert(&glob_conns, 0, mbox, &conn); /* add mbox -> conn mapping */
+ stdmutex_drop(&glob_conns_lock); /* UNLOCK CONNS TAB */
+ }
+ DEBUG(std_stkfprintf(stderr, -1, "FL_connect: ret %d\n", ret));
+ return ret;
+}
+
+/* Destroy a mbox. This fcn recursively reclaims all resources
+ associated with a connection named mbox, in a synchronized, proper
+ fashion and returns whatever SP_disconnect does. If mbox doesn't
+ represent a valid fl connection, it just returns ILLEGAL_SESSION.
+*/
+int FL_disconnect(mailbox mbox) {
+ stdit hit;
+ stdit lit;
+ fl_conn *conn;
+ int ret;
+
+ DEBUG(std_stkfprintf(stderr, 1, "FL_disconnect: mbox %d\n", mbox));
+ stdmutex_grab(&glob_conns_lock); /* LOCK CONNS TAB */
+ if (!stdhash_is_end(&glob_conns, stdhash_find(&glob_conns, &hit, &mbox))) { /* mbox found */
+ conn = *(fl_conn**) stdhash_it_val(&hit);
+ stdhash_erase(&glob_conns, &hit); /* remove mbox -> fl_conn* mapping */
+ stdmutex_drop(&glob_conns_lock); /* UNLOCK CONNS TAB */
+ /* now threads cannot come behind us and make_reservations() on this fl_conn */
+
+ stdmutex_grab(&conn->reserve_lock); /* LOCK RESERVE_LOCK */
+ conn->disconnecting = 1; /* set disconnection indicator */
+ ret = SP_disconnect(mbox); /* unblock any blocked receivers */
+
+ if (conn->reservations != 0) { /* WAIT if connection still in use */
+ DEBUG(std_stkfprintf(stderr, 0, "Waiting for mbox(%d, %p) to no longer be in use: "
+ "%d reservations!\n", mbox, conn, conn->reservations));
+ stdcond_wait(&conn->destroy_cond, &conn->reserve_lock);
+ }
+ assert(conn->reservations == 0); /* should be no more reservations */
+
+ /* now, no other threads are using the connection -> safe to reclaim */
+ DEBUG(std_stkfprintf(stderr, 0, "Mbox(%d, %p) no longer in use: reclaiming!\n", mbox, conn));
+ stdmutex_drop(&conn->reserve_lock); /* UNLOCK RESERVE_LOCK */
+
+ stdmutex_destruct(&conn->reserve_lock); /* recursively destroy fl_conn and sub-structures */
+ stdcond_destruct(&conn->destroy_cond);
+ stdmutex_destruct(&conn->recv_lock);
+ stdmutex_destruct(&conn->conn_lock);
+
+ for (stdhash_begin(&conn->groups, &hit); !stdhash_is_end(&conn->groups, &hit); stdhash_it_next(&hit))
+ free_fl_group(*(fl_group**) stdhash_it_val(&hit));
+ stdhash_destruct(&conn->groups);
+
+ for (stddll_begin(&conn->mess_queue, &lit); !stddll_is_end(&conn->mess_queue, &lit); stddll_it_next(&lit))
+ free_gc_buff_mess(*(gc_buff_mess**) stddll_it_val(&lit));
+ stddll_destruct(&conn->mess_queue);
+
+ free(conn);
+ } else {
+ stdmutex_drop(&glob_conns_lock); /* UNLOCK CONNS TAB */
+ DEBUG(std_stkfprintf(stderr, 0, "FL ILLEGAL_SESSION(%d)\n", mbox));
+ ret = ILLEGAL_SESSION;
+ }
+ DEBUG(std_stkfprintf(stderr, -1, "FL_disconnect: ret %d\n", ret));
+ return ret; /* return SP_disconnect return value */
+}
+
+int FL_join(mailbox mbox, const char *grp) {
+ fl_conn *conn;
+ fl_group *group;
+ int ret;
+
+ DEBUG(std_stkfprintf(stderr, 1, "FL_join: mbox %d, group '%s'\n", mbox, grp));
+ if ((conn = lock_conn(mbox)) != 0) {
+ /* ACHTUNG! You may not join a group when you are already involved with that group */
+ if ((group = get_group(conn, grp)) == 0) {
+ ret = SP_join(mbox, grp);
+ unlock_conn(conn);
+ if (ret != 0) {
+ if (ret == CONNECTION_CLOSED || ret == ILLEGAL_SESSION)
+ FL_disconnect(mbox);
+ else if (ret != ILLEGAL_GROUP)
+ stderr_abort("(%s, %d): mbox %d: group %s: SP_join: unexpected error %d\n",
+ __FILE__, __LINE__, mbox, grp, ret);
+ }
+ } else {
+ ret = ILLEGAL_GROUP;
+ unlock_conn(conn);
+ DEBUG(std_stkfprintf(stderr, 0, "FL ILLEGAL_GROUP('%s', %p) in mstate %d\n", grp,
+ group, group == 0 ? -1 : group->mstate));
+ }
+ } else {
+ DEBUG(std_stkfprintf(stderr, 0, "FL ILLEGAL_SESSION(%d)\n", mbox));
+ ret = ILLEGAL_SESSION;
+ }
+ DEBUG(std_stkfprintf(stderr, -1, "FL_join: ret %d\n", ret));
+ return ret;
+}
+
+int FL_leave(mailbox mbox, const char *grp) {
+ fl_conn *conn;
+ fl_group *group;
+ int ret;
+
+ DEBUG(std_stkfprintf(stderr, 1, "FL_leave: mbox %d, group '%s'\n", mbox, grp));
+ if ((conn = lock_conn(mbox)) != 0) {
+ /* ACHTUNG! If you are a joined member of a fl group only then you can leave it only once! */
+ if ((group = get_group(conn, grp)) != 0 && group->mstate == JOINED) {
+ group->mstate = LEAVING;
+ ret = SP_leave(mbox, grp);
+ unlock_conn(conn);
+ if (ret != 0) {
+ if (ret == CONNECTION_CLOSED || ret == ILLEGAL_SESSION)
+ FL_disconnect(mbox);
+ else if (ret == ILLEGAL_GROUP)
+ stderr_abort("(%s, %d): mbox %d: group %s: SP_leave: ILLEGAL_GROUP\n",
+ __FILE__, __LINE__, mbox, grp);
+ else
+ stderr_abort("(%s, %d): mbox %d: group %s: SP_leave: unexpected error %d\n",
+ __FILE__, __LINE__, mbox, grp, ret);
+ }
+ } else {
+ ret = ILLEGAL_GROUP;
+ unlock_conn(conn);
+ DEBUG(std_stkfprintf(stderr, 0, "FL ILLEGAL_GROUP('%s', %p) in mstate %d\n", grp,
+ group, group == 0 ? -1 : group->mstate));
+ }
+ } else {
+ DEBUG(std_stkfprintf(stderr, 0, "FL ILLEGAL_SESSION(%d)\n", mbox));
+ ret = ILLEGAL_SESSION;
+ }
+ DEBUG(std_stkfprintf(stderr, -1, "FL_leave: ret %d\n", ret));
+ return ret;
+}
+
+int FL_flush(mailbox mbox, const char *grp) {
+ fl_conn *conn;
+ fl_group *group;
+ int ret;
+
+ DEBUG(std_stkfprintf(stderr, 1, "FL_flush: mbox %d, group '%s'\n", mbox, grp));
+ if ((conn = lock_conn(mbox)) != 0) {
+ if ((group = get_group(conn, grp)) != 0)
+ ret = FL_int_flush(conn, group);
+ else {
+ DEBUG(std_stkfprintf(stderr, 0, "FL ILLEGAL_GROUP('%s', 0)\n", grp));
+ ret = ILLEGAL_GROUP;
+ }
+ unlock_conn(conn);
+ if (ret == CONNECTION_CLOSED || ret == ILLEGAL_SESSION)
+ FL_disconnect(mbox);
+ } else {
+ DEBUG(std_stkfprintf(stderr, 0, "FL ILLEGAL_SESSION(%d)\n", mbox));
+ ret = ILLEGAL_SESSION;
+ }
+ DEBUG(std_stkfprintf(stderr, -1, "FL_flush: ret %d\n"));
+ return ret;
+}
+
+int FL_scat_subgroupcast(mailbox mbox, service serv_type, const char *grp,
+ int num_recvrs, char recvrs[][MAX_GROUP_NAME],
+ int16 mess_type, const scatter *scat) {
+ return FL_int_scat_multicast(mbox, serv_type | SUBGROUP_CAST, grp,
+ num_recvrs, recvrs, mess_type, scat);
+}
+
+int FL_subgroupcast(mailbox mbox, service serv_type, const char *grp,
+ int num_recvrs, char recvrs[][MAX_GROUP_NAME],
+ int16 mess_type, int mess_len, const char *mess) {
+ scatter msg;
+
+ msg.num_elements = 1;
+ msg.elements[0].len = mess_len;
+ msg.elements[0].buf = (char*) mess;
+
+ return FL_scat_subgroupcast(mbox, serv_type, grp, num_recvrs, recvrs, mess_type, &msg);
+}
+
+int FL_scat_unicast(mailbox mbox, service serv_type, const char *grp, const char *recvr,
+ int16 mess_type, const scatter *scat) {
+ return FL_scat_subgroupcast(mbox, serv_type, grp, 1, (char(*)[MAX_GROUP_NAME]) recvr,
+ mess_type, scat);
+}
+
+int FL_unicast(mailbox mbox, service serv_type, const char *grp, const char *recvr,
+ int16 mess_type, int mess_len, const char *mess) {
+ scatter msg;
+
+ msg.num_elements = 1;
+ msg.elements[0].len = mess_len;
+ msg.elements[0].buf = (char*) mess;
+
+ return FL_scat_unicast(mbox, serv_type, grp, recvr, mess_type, &msg);
+}
+
+int FL_scat_multicast(mailbox mbox, service serv_type, const char *grp,
+ int16 mess_type, const scatter *scat) {
+ return FL_int_scat_multicast(mbox, serv_type & ~SUBGROUP_CAST, grp, 0, 0, mess_type, scat);
+}
+
+int FL_multicast(mailbox mbox, service serv_type, const char *grp,
+ int16 mess_type, int mess_len, const char *mess) {
+ scatter msg;
+
+ msg.num_elements = 1;
+ msg.elements[0].len = mess_len;
+ msg.elements[0].buf = (char*) mess;
+
+ return FL_scat_multicast(mbox, serv_type, grp, mess_type, &msg);
+}
+
+int FL_scat_receive(mailbox mbox, service *serv_type, char *sender, int max_groups,
+ int *num_groups, char groups[][MAX_GROUP_NAME], int16 *mess_type,
+ int *endian_mismatch, scatter *scat_mess, int *more_messes) {
+ int blocking = (*serv_type & DONT_BLOCK) == 0; /* user not using DONT_BLOCK */
+ gc_recv_mess msg; /* used to contain user's parameters */
+ fl_conn *conn;
+
+ DEBUG(std_stkfprintf(stderr, 1, "FL_scat_receive: mbox %d, service 0x%X, max_groups %d\n",
+ mbox, *serv_type, max_groups));
+
+ if (max_groups < 0) {
+ DEBUG(std_stkfprintf(stderr, 0, "Illegal max_groups!\n"));
+ msg.ret = ILLEGAL_PARAM;
+ } else {
+ /* I use msg rather than the individual parameters to pass user's variables around */
+ msg.mbox = mbox;
+ msg.orig_serv_type = *serv_type;
+ msg.serv_type = serv_type;
+ msg.sender = sender;
+ msg.max_groups = max_groups;
+ msg.num_groups = num_groups;
+ msg.groups = groups;
+ msg.mess_type = mess_type;
+ msg.endian_mismatch = endian_mismatch;
+ msg.scat_mess = scat_mess;
+ msg.delivered = 0; /* does msg contain values to be returned to the user? */
+
+ if ((conn = make_reservation(mbox)) != 0) { /* get conn and make sure it stays valid */
+ /* NEED TO ENFORCE THAT THERE IS ONLY ONE READING THREAD AT A TIME PER CONNECTION, */
+ /* BECAUSE A THREAD COULD BECOME BLOCKED IN SP_recv AND THEN MSGS ARE GENERATED BY */
+ /* ANOTHER THREAD AND PUT INTO THE QUEUE TO BE RETURNED -> THE BLOCKED THREAD */
+ /* WOULDN'T REALIZE IT! SERIALIZATION ON RECV_LOCK ACCOMPLISHES THIS!!!! */
+ if (acquire_recv_lock(conn)) {
+ if (acquire_conn_lock(conn)) { /* get conn lock for so I can read connection info */
+ int have_conn_lock = 1; /* do I have the conn lock? */
+
+ FL_scat_re_receive:
+ msg.vulnerable = 0; /* is the contained message vulnerable? */
+ msg.num_new_grps = 0; /* ensure new_grps and new_msg are empty */
+ msg.new_msg.num_elements = 0; /* used for DROP_RECV msgs (see FL_int_receive) */
+
+ if (stddll_empty(&conn->mess_queue)) { /* empty msg queue */
+ if (blocking || SP_poll(mbox) != 0) { /* blocking allowed, or msg available */
+ have_conn_lock = recv_and_handle(conn, &msg); /* call SP_recv and enter FSA */
+
+ if (msg.num_new_grps != 0) /* reclaim any allocated buffers */
+ free(msg.new_grps);
+
+ if (msg.new_msg.num_elements != 0)
+ free(msg.new_msg.elements[0].buf);
+
+ if (!msg.delivered) { /* if msg doesn't contain returnable values */
+ DEBUG(std_stkfprintf(stderr, 0, "No return msg, going to top of recv loop!\n"));
+ goto FL_scat_re_receive; /* need to try and get a msg again */
+ } else
+ DEBUG(std_stkfprintf(stderr, 0, "Msg contains values for user!\n"));
+ } else {
+ DEBUG(std_stkfprintf(stderr, 0, "Call to receive would have blocked!\n"));
+ msg.ret = WOULD_BLOCK; /* would have blocked */
+ }
+ } else { /* message in queue! */
+ gc_buff_mess *mess; /* get buff mess */
+ stdit lit;
+
+ DEBUG(std_stkfprintf(stderr, 0, "Msg in buffer, reading in to user's parameters\n"));
+ mess = *(gc_buff_mess**) stddll_it_val(stddll_begin(&conn->mess_queue, &lit));
+ if (buffm_to_userm(&msg, mess)) { /* msg copy to user's parameters successful */
+ stddll_erase(&conn->mess_queue, &lit); /* pop mess off of the top of the queue */
+ conn->bytes_queued -= mess->total_size; /* update bytes queued */
+ free_gc_buff_mess(mess); /* destroy the gc_buff_mess */
+ } else
+ DEBUG(std_stkfprintf(stderr, 0, "Msg incompatible with user's parameters!\n"));
+ }
+ if (have_conn_lock) { /* if I didn't lose the conn lock */
+ if (msg.ret >= 0)
+ *more_messes = (int) stddll_size(&conn->mess_queue); /* how many msgs left */
+ release_conn_lock(conn); /* let go of the conn lock */
+ }
+ } else {
+ DEBUG(std_stkfprintf(stderr, 0, "Couldn't acquire conn lock -> disconnecting!\n"));
+ msg.ret = ILLEGAL_SESSION;
+ }
+ release_recv_lock(conn);
+ } else {
+ DEBUG(std_stkfprintf(stderr, 0, "Couldn't acquire recv lock -> disconnecting!\n"));
+ msg.ret = ILLEGAL_SESSION;
+ }
+ cancel_reservation(conn);
+ } else {
+ DEBUG(std_stkfprintf(stderr, 0, "Couldn't get a reservation\n"));
+ msg.ret = ILLEGAL_SESSION;
+ }
+ DEBUG( /* only execute if statment when DEBUGGING */
+ if (msg.ret >= 0) {
+ int i;
+
+ std_stkfprintf(stderr, 0, "Groups(%d):\n", *num_groups);
+ for (i = 0; i < *num_groups; ++i)
+ std_stkfprintf(stderr, 0, "\t'%s'\n", groups[i]);
+ });
+ if (msg.ret == CONNECTION_CLOSED || msg.ret == ILLEGAL_SESSION)
+ FL_disconnect(mbox);
+ }
+ DEBUG(std_stkfprintf(stderr, -1, "FL_scat_receive: ret %d, mbox %d, service 0x%X, sender '%s'"
+ ", num_groups %d, mess_type %d, endian %d, more_messes %d\n",
+ msg.ret, mbox, *serv_type, msg.ret >= 0 ? sender : "", *num_groups,
+ *mess_type, *endian_mismatch, *more_messes));
+ return msg.ret;
+}
+
+int FL_receive(mailbox mbox, service *serv_type, char *sender, int max_groups,
+ int *num_groups, char groups[][MAX_GROUP_NAME], int16 *mess_type,
+ int *endian_mismatch, int max_mess_len, char *mess, int *more_messes) {
+ scatter msg;
+
+ msg.num_elements = 1;
+ msg.elements[0].len = max_mess_len;
+ msg.elements[0].buf = (char*) mess;
+
+ return FL_scat_receive(mbox, serv_type, sender, max_groups, num_groups, groups,
+ mess_type, endian_mismatch, &msg, more_messes);
+}
+
+int FL_more_msgs(mailbox mbox) {
+ fl_conn *conn;
+ int ret;
+
+ DEBUG(std_stkfprintf(stderr, 1, "FL_more_msgs: mbox %d\n", mbox));
+ if ((conn = lock_conn(mbox)) != 0) {
+ ret = (int) stddll_size(&conn->mess_queue);
+ unlock_conn(conn);
+ } else {
+ DEBUG(std_stkfprintf(stderr, 0, "FL ILLEGAL_SESSION(%d)\n", mbox));
+ ret = ILLEGAL_SESSION;
+ }
+ DEBUG(std_stkfprintf(stderr, -1, "FL_more_msgs: ret %d\n", ret));
+ return ret;
+}
+
+int FL_poll(mailbox mbox) {
+ fl_conn *conn;
+ int ret;
+
+ DEBUG(std_stkfprintf(stderr, 1, "FL_poll: mbox %d\n", mbox));
+ if ((conn = lock_conn(mbox)) != 0) {
+ ret = conn->bytes_queued + SP_poll(mbox);
+ unlock_conn(conn);
+ } else {
+ DEBUG(std_stkfprintf(stderr, 0, "FL ILLEGAL_SESSSION(%d)\n", mbox));
+ ret = ILLEGAL_SESSION;
+ }
+ DEBUG(std_stkfprintf(stderr, -1, "FL_poll: ret %d\n", ret));
+ return ret;
+}
+
+void FL_error(int err) {
+ switch (err) {
+ case ILLEGAL_PARAM:
+ printf("FL_error: (%d) Illegal parameter (eg a negative size) passed to a function.\n", err);
+ break;
+ case WOULD_BLOCK:
+ printf("FL_error: (%d) Function call would have blocked.\n", err);
+ break;
+ case ILLEGAL_MESSAGE_TYPE:
+ printf("FL_error: (%d) Illegal message type (int16) used, "
+ "value < FL_MIN_LEGAL_MESS_TYPE(%d).\n", err, FL_MIN_LEGAL_MESS_TYPE);
+ break;
+ case ILLEGAL_STATE:
+ printf("FL_error: (%d) Function call peformed in a prohibited state.\n", err);
+ break;
+ case ILLEGAL_RECEIVERS:
+ printf("FL_error: (%d) Illegal receivers specified.\n", err);
+ break;
+ default:
+ SP_error(err);
+ break;
+ }
+}
+
+/************************************ private interface ****************************************/
+
+/* structure routines */
+/* When I initially create a group structure I set it up to be ready to insert in to a groups */
+/* hash of a connection. I set mstate to JOINING indicating this group hasn't installed this */
+/* connection as a member yet. I set vstate to AGREE, which will trigger auto-flush oks from */
+/* this joining connection on SP memb events, until it finally gets installed. I create a */
+/* fl_view with only the connection in fl_view->membs. I do this for three reasons: (1) */
+/* sp_view and fl_view are always pointing at valid views now (no special checks for null), */
+/* (2) in case this connection is joined through a NETWORK event, it will report the proper */
+/* singleton vs set, and (3) this also gives me correct behavior on delivery of user's data */
+/* msgs (is sender in my current fl view?). */
+static fl_group *create_fl_group(const char *conn_name, const char *group_name) {
+ fl_group *group;
+ group_id gid = {};
+
+ DEBUG(std_stkfprintf(stderr, 1, "create_fl_group: conn_name '%s', group '%s'\n",
+ conn_name, group_name));
+
+ if ((group = (fl_group*) calloc(1, sizeof(fl_group))) == 0)
+ stderr_abort("(%s, %d): calloc(1, %u)\n", __FILE__, __LINE__, sizeof(fl_group));
+
+ strncpy(group->group, group_name, MAX_GROUP_NAME);
+ group->mstate = JOINING; /* not a JOINED member yet */
+ group->vstate = AGREE; /* when I handle_next_memb_change first time -> sends FLUSH_OK */
+ group->curr_change = 0;
+ group->sp_view = group->fl_view = create_view(gid); /* put an empty view to start */
+ fill_view(group->fl_view, 0, 1, (char(*)[MAX_GROUP_NAME]) conn_name, 0); /* insert self */
+ group->fl_view->in_trans_memb = 1; /* don't deliver any trans sigs until a member */
+ group->flush_recvs = 0;
+ stddll_construct(&group->mess_queue, sizeof(gc_buff_mess*));
+ stddll_construct(&group->memb_queue, sizeof(sp_memb_change*));
+ stdhash_construct(&group->pmemb_hash, sizeof(group_id), sizeof(sp_memb_change*),
+ group_id_cmp, group_id_hashcode, 0); /* use default key fcns for group_id */
+ DEBUG(std_stkfprintf(stderr, -1, "create_fl_group: group('%s', %p)\n", group->group, group));
+ return group;
+}
+
+static void free_fl_group(fl_group *group) {
+ stdit lit;
+ stdit hit;
+
+ DEBUG(std_stkfprintf(stderr, 1, "free_fl_group: group('%s', %p)\n", group->group, group));
+ /* sp view is either equal to fl_view or points in to one of the sp_memb_change's */
+ free_view(group->fl_view);
+
+ for (stddll_begin(&group->mess_queue, &lit); !stddll_is_end(&group->mess_queue, &lit); stddll_it_next(&lit))
+ free_gc_buff_mess(*(gc_buff_mess**) stddll_it_val(&lit));
+ stddll_destruct(&group->mess_queue);
+
+ /* curr_change and all sp_memb_changes contained in memb_queue are also in pmemb */
+ stddll_destruct(&group->memb_queue);
+
+ for (stdhash_begin(&group->pmemb_hash, &hit); !stdhash_is_end(&group->pmemb_hash, &hit); stdhash_it_next(&hit))
+ free_sp_memb_change(*(sp_memb_change**) stdhash_it_val(&hit));
+ stdhash_destruct(&group->pmemb_hash);
+
+ free(group);
+ DEBUG(std_stkfprintf(stderr, -1, "free_fl_group\n"));
+}
+
+static void free_gc_buff_mess(gc_buff_mess *msg) {
+ DEBUG(std_stkfprintf(stderr, 1, "free_gc_buff_mess: m %p, serv 0x%X\n", msg, msg->serv_type));
+ if (msg->num_groups)
+ free(msg->groups);
+
+ if (msg->mess_len)
+ free(msg->mess);
+
+ free(msg);
+ DEBUG(std_stkfprintf(stderr, -1, "free_gc_buff_mess\n"));
+}
+
+static sp_memb_change *create_sp_memb_change(group_id gid) {
+ sp_memb_change *spc;
+
+ DEBUG(std_stkfprintf(stderr, 1, "create_sp_memb_change: gid %d %d %d\n",
+ gid.id[0], gid.id[1], gid.id[2]));
+
+ if ((spc = (sp_memb_change*) calloc(1, sizeof(sp_memb_change))) == 0)
+ stderr_abort("(%s, %d): calloc(1, %u)\n", __FILE__, __LINE__, sizeof(sp_memb_change));
+
+ spc->memb_info = create_view(gid);
+ spc->memb_mess_recvd = 0;
+ spc->memb_change_age = 0;
+ spc->delta_member = 0;
+ stdhash_construct(&spc->flok_senders, MAX_GROUP_NAME, 0,
+ group_name_cmp, group_name_hashcode, 0);
+ spc->gid = gid;
+
+ DEBUG(std_stkfprintf(stderr, -1, "create_sp_memb_change: ret %p, gid %d %d %d\n", spc,
+ spc->memb_info->gid.id[0], spc->memb_info->gid.id[1],
+ spc->memb_info->gid.id[2]));
+ return spc;
+}
+
+static void free_sp_memb_change(sp_memb_change *spc) {
+ DEBUG(std_stkfprintf(stderr, 1, "free_sp_memb_change: spc %p\n", spc));
+
+ if (spc->memb_info != 0)
+ free_view(spc->memb_info);
+
+ if (spc->delta_member != 0)
+ free(spc->delta_member);
+
+ stdhash_destruct(&spc->flok_senders);
+ free(spc);
+ DEBUG(std_stkfprintf(stderr, -1, "free_sp_memb_change\n"));
+}
+
+static view *create_view(group_id gid) {
+ view *v;
+
+ DEBUG(std_stkfprintf(stderr, 1, "create_view: gid %d %d %d\n",
+ gid.id[0], gid.id[1], gid.id[2]));
+
+ if ((v = (view*) calloc(1, sizeof(view))) == 0)
+ stderr_abort("(%s, %d): calloc(1, %u)\n", sizeof(view));
+
+ v->gid = gid;
+ v->memb_type = 0;
+ v->in_trans_memb = 0;
+ v->orig_num_membs = 0;
+ v->membs_names = 0;
+ stdhash_construct(&v->orig_membs, sizeof(char*), 0,
+ group_name_ptr_cmp, group_name_ptr_hashcode, 0);
+ stdhash_construct(&v->curr_membs, sizeof(char*), 0,
+ group_name_ptr_cmp, group_name_ptr_hashcode, 0);
+ v->my_index = -1;
+
+ DEBUG(std_stkfprintf(stderr, -1, "create_view: ret %p\n", v));
+ return v;
+}
+
+static void free_view(view *v) {
+ DEBUG(std_stkfprintf(stderr, 1, "free_view: view %p, gid %d %d %d\n", v, v->gid.id[0],
+ v->gid.id[1], v->gid.id[2]));
+ if (v->orig_num_membs != 0)
+ free(v->membs_names);
+ stdhash_destruct(&v->orig_membs);
+ stdhash_destruct(&v->curr_membs);
+ free(v);
+ DEBUG(std_stkfprintf(stderr, -1, "free_view\n"));
+}
+
+static void fill_view(view *v, service memb_type, int num_membs,
+ char (*membs)[MAX_GROUP_NAME], int16 index) {
+ size_t byte_size = num_membs * MAX_GROUP_NAME;
+
+ DEBUG(std_stkfprintf(stderr, 1, "fill_view: view %p, serv 0x%X, num_membs %d, index %d\n",
+ v, memb_type, num_membs, index));
+
+ assert(num_membs > 0 && index >= 0 && index < num_membs && v->orig_num_membs == 0 &&
+ stdhash_empty(&v->orig_membs) && stdhash_empty(&v->orig_membs));
+
+ v->memb_type = memb_type;
+ v->orig_num_membs = num_membs;
+
+ if ((v->membs_names = (char(*)[MAX_GROUP_NAME]) malloc(byte_size)) == 0)
+ stderr_abort("(%s, %d): malloc (%d)\n", __FILE__, __LINE__, byte_size);
+ memcpy(v->membs_names, membs, byte_size);
+
+ for (membs = v->membs_names; num_membs--; ++membs) /* I set membs = v->membs_names!!! */
+ stdhash_insert(&v->orig_membs, 0, &membs, 0), stdhash_insert(&v->curr_membs, 0, &membs, 0);
+
+ v->my_index = index;
+ DEBUG(std_stkfprintf(stderr, -1, "fill_view: view %p, serv 0x%X, num_membs %lu, index %d\n",
+ v, v->memb_type, stdhash_size(&v->orig_membs), v->my_index));
+}
+
+/* Looks up a fl_conn "named" mbox, gets an outstanding reservation on
+ it and acquires its conn_lock. Returns the valid locked fl_conn* on
+ success or 0 on failure -> ILLEGAL_SESSION.
+*/
+static fl_conn *lock_conn(mailbox mbox) {
+ fl_conn *conn;
+
+ if ((conn = make_reservation(mbox)) == 0)
+ return 0;
+
+ if (!acquire_conn_lock(conn)) /* error -> disconnecting */
+ return cancel_reservation(conn), (fl_conn*) 0;
+
+ return conn;
+}
+
+/* Unlock a connection successfully locked with lock_conn. */
+static void unlock_conn(fl_conn *conn) {
+ release_conn_lock(conn);
+ cancel_reservation(conn);
+}
+
+/* Get a pointer to a fl_conn "named" mbox and "acquire an outstanding
+ reservation" by increasing a reservation counter. A successful
+ return *ONLY* entitles the caller to make subsequent calls to
+ acquire/release_conn_lock and acquire/release_recv_lock on the
+ returned fl_conn. Each successful call to make_reservation MUST be
+ matched by a call to cancel_reservation on the returned
+ fl_conn. This outstanding reservation only ensures that the
+ returned fl_conn is valid between a successful call to
+ make_reservation and the subsequent call to cancel_reservation on
+ the returned fl_conn. If mbox is not a valid fl connection this fcn
+ fails and returns 0. NOTE: it is imperative that this fcn acquires
+ the reserve lock before releasing the glob_conns_lock.
+*/
+static fl_conn *make_reservation(mailbox mbox) {
+ stdit hit;
+ fl_conn *conn;
+
+ stdmutex_grab(&glob_conns_lock); /* LOCK CONNS TAB */
+
+ if (stdhash_is_end(&glob_conns, stdhash_find(&glob_conns, &hit, &mbox))) /* no such mbox */
+ return stdmutex_drop(&glob_conns_lock), (fl_conn*) 0; /* UNLOCK CONNS TAB */
+
+ conn = *(fl_conn**) stdhash_it_val(&hit);
+ stdmutex_grab(&conn->reserve_lock); /* LOCK RESERVE_LOCK */
+ stdmutex_drop(&glob_conns_lock); /* UNLOCK CONNS TAB */
+ if (!conn->disconnecting) /* check if disconnecting */
+ ++conn->reservations; /* increment reservation cnt */
+ stdmutex_drop(&conn->reserve_lock); /* UNLOCK RESERVE_LOCK */
+
+ return !conn->disconnecting ? conn : 0;
+}
+
+/* Relinquish a reservation this thread owns on a connection. */
+static void cancel_reservation(fl_conn *conn) {
+ stdmutex_grab(&conn->reserve_lock); /* LOCK RESERVE_LOCK */
+ assert(conn->reservations != 0);
+ if (--conn->reservations == 0 && conn->disconnecting) /* decrement reservation cnt */
+ stdcond_wake_all(&conn->destroy_cond); /* signal disconnector thread */
+ stdmutex_drop(&conn->reserve_lock); /* UNLOCK RESERVE_LOCK */
+}
+
+/* Acquire a connection's conn lock. The caller must have an
+ outstanding reservation on conn. Once this lock is acquired, a
+ thread can freely examine and/or modify the data in conn as
+ indicated in <fl_p.h>. Returns 0 on failure -> ILLEGAL_SESSION.
+ Each successful call made by a thread on a particular conn must be
+ matched with a call to release_conn_lock to allow that conn to be
+ used by other threads. A thread may not make a second call to
+ acquire_conn_lock after a successful return on a particular conn
+ without an intervening call to release_conn_lock on that conn to
+ prevent self-deadlocks.
+*/
+static int acquire_conn_lock(fl_conn *conn) {
+ stdmutex_grab(&conn->conn_lock); /* LOCK CONN_LOCK */
+ if (conn->disconnecting) /* conn is disconnecting */
+ return stdmutex_drop(&conn->conn_lock), 0;
+
+ return 1;
+}
+
+/* Release a connection's conn lock. The caller must have an
+ oustanding reservation on release. The calling thread must also
+ have made a successful call to acquire_conn_lock on release without
+ any intervening calls to release_conn_lock on release.
+*/
+static void release_conn_lock(fl_conn *release) {
+ stdmutex_drop(&release->conn_lock); /* UNLOCK CONN_LOCK */
+}
+
+/* Acquire a connection's recv lock. The caller must have an
+ outstanding reservation on conn. Once this lock is acquired, a
+ thread can proceed into the body of FL_scat_recv. Returns 0
+ on failure -> ILLEGAL_SESSION. Each successful call made by a
+ thread on a particular conn must be matched with a call to
+ release_recv_lock to allow that conn to be used by other threads. A
+ thread may not make a second call to acquire_recv_lock after a
+ successful return on a particular conn without an intervening call
+ to release_recv_lock on that conn to prevent self-deadlocks. NOTE
+ THAT THIS LOCK DOES NOT GIVE A THREAD ANY RIGHTS TO EXAMINE OR
+ MODIFY _ANY_ OF THE CONNECTION'S DATA!! IT ONLY ALLOWS A THREAD TO
+ ENTER THE BODY OF FL_scat_recv!!
+*/
+static int acquire_recv_lock(fl_conn *conn) {
+ stdmutex_grab(&conn->recv_lock); /* LOCK RECV_LOCK */
+ if (conn->disconnecting) /* conn is disconnecting */
+ return stdmutex_drop(&conn->recv_lock), 0;
+
+ return 1;
+}
+
+/* Release a connection's recv lock. The caller must have an
+ oustanding reservation on release. The calling thread must have
+ made a successful call to acquire_recv_lock on release without any
+ intervening calls to release_recv_lock on release.
+*/
+static void release_recv_lock(fl_conn *release) {
+ stdmutex_drop(&release->recv_lock); /* UNLOCK RECV_LOCK */
+}
+
+static fl_group *add_group(fl_conn *conn, const char *group_name) {
+ fl_group *group;
+ char *group_name_ptr;
+
+ DEBUG(std_stkfprintf(stderr, 1, "add_group: mbox(%d, %p), group '%s'\n",
+ conn->mbox, conn, group_name));
+ group = create_fl_group(conn->private, group_name);
+ group_name_ptr = group->group;
+ stdhash_insert(&conn->groups, 0, &group_name_ptr, &group);
+ DEBUG(std_stkfprintf(stderr, -1, "add_group: return group('%s', %p)\n", group->group, group));
+ return group;
+}
+
+static void remove_group(fl_conn *conn, fl_group *group) {
+ stdit hit;
+ char *group_name_ptr;
+
+ DEBUG(std_stkfprintf(stderr, 1, "remove_group: mbox(%d, %p), group ('%s', %p)\n",
+ conn->mbox, conn, group->group, group));
+ group_name_ptr = group->group;
+ stdhash_find(&conn->groups, &hit, &group_name_ptr);
+ assert(!stdhash_is_end(&conn->groups, &hit) && group == *(fl_group**) stdhash_it_val(&hit));
+ stdhash_erase(&conn->groups, &hit);
+ free_fl_group(group);
+ DEBUG(std_stkfprintf(stderr, -1, "remove_group\n"));
+}
+
+static fl_group *get_group(fl_conn *conn, const char *grp) {
+ stdit hit;
+
+ return (!stdhash_is_end(&conn->groups, stdhash_find(&conn->groups, &hit, &grp)) ?
+ *(fl_group**) stdhash_it_val(&hit) : 0);
+}
+
+/* Pass null for bm if the data to be delivered is already in um. */
+/* Otherwise fill out a gc_buff_mess with the appropriate data and indicate whether */
+/* that buff mess is dynamically allocated or not. If it is, then if the message needs to */
+/* be buffered, then bm itself is buffered. If the dynamically allocated message doesn't */
+/* need to be buffered, then the msg is reclaimed (freed) before returning to the caller. */
+/* Returns a pointer to the gc_buff_mess that was buffered, null if none was buffered. */
+/* um can be null if you force buffering somehow. */
+static gc_buff_mess *deliver(fl_conn *conn, gc_recv_mess *um, gc_buff_mess *bm,
+ int bm_alloced) {
+ gc_buff_mess *ret;
+
+ DEBUG(std_stkfprintf(stderr, 1, "deliver: mbox(%d, %p), um %p, bm %p, bm_alloced %d\n",
+ conn->mbox, conn, um, bm, (int) bm_alloced));
+ if (!um->delivered && stddll_empty(&conn->mess_queue) && (bm == 0 || buffm_to_userm(um, bm))) {
+ DEBUG(std_stkfprintf(stderr, 0, "able to deliver to user's parameters!\n"));
+ um->delivered = 1;
+ ret = 0;
+ if (bm != 0 && bm_alloced) {
+ DEBUG(std_stkfprintf(stderr, 0, "bm was alloc'ed -> freeing %p\n", bm));
+ free_gc_buff_mess(bm);
+ }
+ } else {
+ DEBUG(std_stkfprintf(stderr, 0, "UNABLE to deliver to user's parameters: BUFFERING!\n"));
+ if (bm == 0 || !bm_alloced) { /* if I need to alloc a gc_buff_mess */
+ if ((ret = (gc_buff_mess*) malloc(sizeof(gc_buff_mess))) == 0)
+ stderr_abort("(%s, %d): malloc(%u)\n", __FILE__, __LINE__, sizeof(gc_buff_mess));
+
+ if (bm == 0) { /* if data was contained in user parameters */
+ DEBUG(std_stkfprintf(stderr, 0, "data was in user's parameters copying to buff mess\n"));
+ userm_to_buffm(ret, um);
+ } else { /* else data was contained in bm */
+ DEBUG(std_stkfprintf(stderr, 0, "data was in bm, copying in to buff mess\n"));
+ *ret = *bm;
+ }
+ } else { /* buff mess was already alloc'ed and filled out */
+ DEBUG(std_stkfprintf(stderr, 0, "data in bm, but alloc'ed: stealing bm for buffer\n"));
+ ret = bm;
+ }
+ /* calculate total size of buffered message */
+ ret->total_size = sizeof(gc_buff_mess) + ret->num_groups * MAX_GROUP_NAME + ret->mess_len;
+ stddll_push_back(&conn->mess_queue, &ret);
+ conn->bytes_queued += ret->total_size;
+ }
+ DEBUG(std_stkfprintf(stderr, -1, "deliver: ret buff mess %p\n", ret));
+ return ret;
+}
+
+static gc_buff_mess *deliver_trans_sig(fl_conn *conn, gc_recv_mess *um, char *group) {
+ gc_buff_mess trans_sig = {}, *ret; /* sets everything to zero */
+
+ DEBUG(std_stkfprintf(stderr, 1, "deliver_trans_sig: mbox(%d, %p), group '%s'\n",
+ conn->mbox, conn, group));
+ trans_sig.mbox = conn->mbox;
+ trans_sig.serv_type = TRANSITION_MESS;
+ strncpy(trans_sig.sender, group, MAX_GROUP_NAME);
+ ret = deliver(conn, um, &trans_sig, 0);
+
+ DEBUG(std_stkfprintf(stderr, -1, "deliver_trans_sig: ret buff mess %p\n", ret));
+ return ret;
+}
+
+static gc_buff_mess *deliver_flush_req(fl_conn *conn, gc_recv_mess *um, char *group) {
+ gc_buff_mess flush_req = {}, *ret; /* sets everything to zero */
+
+ DEBUG(std_stkfprintf(stderr, 1, "deliver_flush_req: mbox(%d, %p), group '%s'\n",
+ conn->mbox, conn, group));
+ flush_req.mbox = conn->mbox;
+ flush_req.serv_type = FLUSH_REQ_MESS;
+ strncpy(flush_req.sender, group, MAX_GROUP_NAME);
+ ret = deliver(conn, um, &flush_req, 0);
+
+ DEBUG(std_stkfprintf(stderr, -1, "deliver_flush_req: ret buff mess %p\n", ret));
+ return ret;
+}
+
+/* copy info from buffered msg to a user's parameters and make it presentable to the user */
+/* return 1 on successful copy to user, 0 on failure (incompatible user parameters) */
+static int buffm_to_userm(gc_recv_mess *um, const gc_buff_mess *bm) {
+ int ret = 1;
+ scatp pos;
+ long err;
+
+ DEBUG(std_stkfprintf(stderr, 1, "buffm_to_userm: um %p, bm %p, serv 0x%X, sender '%s'\n",
+ um, bm, bm->serv_type, bm->sender));
+
+ if (scatp_begin(&pos, um->scat_mess) == 0 &&
+ (err = scatp_cpy2(&pos, bm->mess, bm->mess_len)) >= 0) { /* user's scatter is legal */
+ um->mbox = bm->mbox;
+ *um->serv_type = bm->serv_type;
+ *um->mess_type = bm->mess_type;
+
+ if (err == bm->mess_len && bm->num_groups <= um->max_groups) { /* no buffer problem */
+ *um->num_groups = bm->num_groups;
+ strncpy(um->sender, bm->sender, MAX_GROUP_NAME);
+ memcpy(um->groups, bm->groups,
+ (*um->num_groups >= 0 ? *um->num_groups : um->max_groups) * MAX_GROUP_NAME);
+ *um->endian_mismatch = bm->endian_mismatch;
+ um->ret = bm->mess_len;
+ } else { /* buffer problem */
+ DEBUG(std_stkfprintf(stderr, 0, "User buffer problem!\n"));
+ if ((um->orig_serv_type & DROP_RECV) == 0) { /* user didn't request DROP_RECV */
+ DEBUG(std_stkfprintf(stderr, 0, "User didn't request DROP_RECV\n"));
+ if (err != bm->mess_len) {
+ um->ret = BUFFER_TOO_SHORT;
+ *um->endian_mismatch = -bm->mess_len;
+ } else
+ *um->endian_mismatch = 0;
+
+ if (bm->num_groups > um->max_groups) {
+ um->ret = GROUPS_TOO_SHORT;
+ *um->num_groups = -bm->num_groups;
+ } else
+ *um->num_groups = 0;
+
+ ret = 0; /* error: buffer problem and DROP_RECV not requested */
+ } else { /* user requested DROP_RECV */
+ DEBUG(std_stkfprintf(stderr, 0, "User DID request DROP_RECV!\n"));
+ if (err != bm->mess_len)
+ um->ret = BUFFER_TOO_SHORT;
+
+ if (bm->num_groups > um->max_groups) {
+ um->ret = GROUPS_TOO_SHORT;
+ *um->num_groups = -bm->num_groups;
+ } else
+ *um->num_groups = bm->num_groups;
+
+ strncpy(um->sender, bm->sender, MAX_GROUP_NAME);
+ memcpy(um->groups, bm->groups,
+ (*um->num_groups >= 0 ? *um->num_groups : um->max_groups) * MAX_GROUP_NAME);
+ *um->endian_mismatch = bm->endian_mismatch;
+ }
+ }
+ } else { /* error: user's scat msg is illegal */
+ um->ret = ILLEGAL_MESSAGE;
+ ret = 0;
+ }
+ DEBUG(std_stkfprintf(stderr, -1, "buffm_to_userm: um %p, bm %p, serv 0x%X, sender '%s'\n",
+ um, bm, bm->serv_type, bm->sender));
+ return ret;
+}
+
+/* copy info from a user's parameters to a buffered message */
+static void userm_to_buffm(gc_buff_mess *bm, const gc_recv_mess *um) {
+ bm->mbox = um->mbox;
+ bm->serv_type = *um->serv_type;
+ strncpy(bm->sender, um->sender, MAX_GROUP_NAME);
+ bm->mess_type = *um->mess_type;
+ bm->endian_mismatch = *um->endian_mismatch;
+
+ DEBUG(std_stkfprintf(stderr, 1, "userm_to_buffm: ret %d, vuln %d, delivered %d, serv 0x%X, "
+ "sender '%s'\n", um->ret, (int) um->vulnerable, (int) um->delivered,
+ *um->serv_type, um->sender));
+ /* if um isn't an error without a msg body and groups */
+ if (um->ret >= 0 || um->ret == GROUPS_TOO_SHORT || um->ret == BUFFER_TOO_SHORT) {
+ scatter *scat;
+ size_t groups_size;
+ scatp um_pos;
+ char *groups;
+ long err;
+
+ /* get groups info out of the msg: either in um->groups or um->new_grps */
+ get_groups_info(um, &bm->num_groups, (char(**)[MAX_GROUP_NAME]) &groups);
+ if ((groups_size = bm->num_groups * MAX_GROUP_NAME) != 0) {
+ if ((bm->groups = (char(*)[MAX_GROUP_NAME]) malloc(groups_size)) == 0)
+ stderr_abort("(%s, %d): malloc(%u)\n", __FILE__, __LINE__, groups_size);
+ memcpy(bm->groups, groups, groups_size);
+ } else
+ bm->groups = 0;
+
+ /* get scat info out of the msg: either in um->scat_mess or um->new_msg */
+ get_scat_info(um, &bm->mess_len, &scat);
+ if (bm->mess_len != 0) {
+ if ((bm->mess = (char*) malloc(bm->mess_len)) == 0)
+ stderr_abort("(%s, %d): malloc(%d)\n", __FILE__, __LINE__, bm->mess_len);
+
+ err = scatp_begin(&um_pos, scat);
+ assert(err == 0);
+ err = scatp_cpy1(bm->mess, &um_pos, bm->mess_len);
+ assert(err == bm->mess_len);
+ } else
+ bm->mess = 0;
+ } else {
+ stderr_abort("not sure about this path: if ever triggered think about it\n");
+ DEBUG(std_stkfprintf(stderr, 0, "copying an error message to a buffm: ret %d\n", um->ret));
+ bm->num_groups = 0;
+ bm->groups = 0;
+ bm->mess_len = um->ret;
+ bm->mess = 0;
+ }
+ DEBUG(std_stkfprintf(stderr, -1, "userm_to_buffm\n"));
+}
+
+static int FL_int_flush(fl_conn *conn, fl_group *group) {
+ int ret;
+
+ DEBUG(std_stkfprintf(stderr, 1, "FL_int_flush: mbox(%d, %p), group ('%s', %p)\n",
+ conn->mbox, conn, group->group, group));
+ /* ACHTUNG! You must be a non-leaving, AUTHORIZE member of a fl group to flush it */
+ if (group->mstate != LEAVING) {
+ if (group->vstate == AUTHORIZE) {
+ DEBUG(std_stkfprintf(stderr, 0, "Going to state AGREE: Actually sending "
+ "the FLUSH_OK_MESS!\n"));
+ group->vstate = AGREE;
+ ret = SP_multicast(conn->mbox, FIFO_MESS, group->group, FLUSH_OK_MESS,
+ sizeof(group_id), (char*) &group->curr_change->memb_info->gid);
+
+ if (ret != sizeof(group_id)) {
+ if (ret != CONNECTION_CLOSED && ret != ILLEGAL_SESSION)
+ stderr_abort("(%s, %d): mbox %d: group %s: SP_multicast: unexpected error(%d)\n",
+ __FILE__, __LINE__, conn->mbox, group->group, ret);
+ } else
+ ret = 0; /* set to zero -> success */
+ } else {
+ DEBUG(std_stkfprintf(stderr, 0, "Group not in AUTHORIZE state!\n"));
+ ret = ILLEGAL_STATE;
+ }
+ } else {
+ DEBUG(std_stkfprintf(stderr, 0, "Group is in LEAVING state!\n"));
+ ret = ILLEGAL_GROUP;
+ }
+ DEBUG(std_stkfprintf(stderr, -1, "FL_int_flush: ret %d\n", ret));
+ return ret;
+}
+
+static int FL_int_scat_multicast(mailbox mbox, service serv_type, const char *grp,
+ int num_recvrs, char recvrs[][MAX_GROUP_NAME],
+ int16 mess_type, const scatter *user_scat) {
+ fl_conn *conn;
+ fl_group *group = 0;
+ int grp_not_priv; /* is grp not a private group ? */
+ int ret, i;
+
+ int fix_scat = 0; /* did I modify the user's scat? */
+ scatter *scat = (scatter*) user_scat; /* get rid of const warnings */
+ scat_element senders_elem; /* for copy of user's element I might overwrite */
+ char copy_buf[sizeof(group_id) + sizeof(int16) + MAX_GROUP_NAME]; /* append buffer */
+ char *curr_ptr = copy_buf, *key; /* current pos in append buffer */
+
+ DEBUG( /* debug print out user's parameters */
+ std_stkfprintf(stderr, 1, "FL_int_scat_multicast: mbox %d, serv 0x%X, grp '%s', "
+ "mess_type %d, mess_len %ld\n", mbox, serv_type, grp,
+ mess_type, scat_capacity(scat));
+ std_stkfprintf(stderr, 0, "Recvrs(%d):\n", num_recvrs);
+ for (i = 0; i < num_recvrs; ++i) {
+ std_stkfprintf(stderr, 0, "\t'%s'\n", recvrs[i]);
+ });
+
+ if ((conn = lock_conn(mbox)) != 0) { /* LOCK CONNECTION */
+ if (num_recvrs < 0) { /* num_recvrs? */
+ DEBUG(std_stkfprintf(stderr, 0, "Illegal num_recvrs %d!\n", num_recvrs));
+ ret = ILLEGAL_PARAM;
+ } else if (IS_ILLEGAL_SEND_STYPE(serv_type)) { /* used reserved bits in serv_type */
+ DEBUG(std_stkfprintf(stderr, 0, "Illegal use of reserved service bits\n"));
+ ret = ILLEGAL_SERVICE;
+ } else if (IS_ILLEGAL_SEND_MTYPE(mess_type)) { /* used a reserved mess type */
+ DEBUG(std_stkfprintf(stderr, 0, "Illegal use of reserved message type!\n"));
+ ret = ILLEGAL_MESSAGE_TYPE;
+ } else if (scat->num_elements < 0 || scat->num_elements > FL_MAX_SCATTER_ELEMENTS) {
+ DEBUG(std_stkfprintf(stderr, 0, "Illegal scatter num_elements %d\n", scat->num_elements));
+ ret = ILLEGAL_MESSAGE;
+ } else if ((grp_not_priv = !is_private_group(grp)) && /* I allow sends to private groups */
+ ((group = get_group(conn, grp)) == 0 || group->mstate != JOINED)) {
+ DEBUG(std_stkfprintf(stderr, 0, "Illegal send to group of which not a member!\n"));
+ ret = ILLEGAL_GROUP; /* not a member of that group */
+ } else if (grp_not_priv && group->vstate == AGREE) {
+ DEBUG(std_stkfprintf(stderr, 0, "Illegal send: state AGREE: group '%s'!\n", group->group));
+ ret = ILLEGAL_STATE;
+ } else { /* attempt to send msg: still need to check on recvrs for subgroup_casts */
+ if (grp_not_priv && is_vulnerable_mess(group, serv_type)) { /* mess type, current fl vid */
+ DEBUG(std_stkfprintf(stderr, 0, "Sending a vulnerable message!\n"));
+ fix_scat = 1;
+ memcpy(curr_ptr, &mess_type, sizeof(int16));
+ curr_ptr += sizeof(int16);
+ memcpy(curr_ptr, &group->fl_view->gid, sizeof(group_id));
+ curr_ptr += sizeof(group_id);
+ mess_type = VULNERABLE_MESS;
+ }
+ if (!Is_subgroup_mess(serv_type)) { /* normal multicast */
+ if (fix_scat) { /* actually change the scatter to reflect appendings */
+ senders_elem = scat->elements[scat->num_elements]; /* save user's element */
+ scat->elements[scat->num_elements].buf = copy_buf;
+ scat->elements[scat->num_elements].len = (int) (curr_ptr - copy_buf);
+ ++scat->num_elements;
+ }
+ ret = SP_scat_multicast(mbox, serv_type, grp, mess_type, scat);
+ } else { /* subgroup multicast */
+ DEBUG(std_stkfprintf(stderr, 0, "Sending a subgroupcast!\n"));
+ if (grp_not_priv) { /* check all recvrs were members of my current fl view */
+ stdhash *orig_membs = &group->fl_view->orig_membs;
+ stdit find;
+
+ for (i = 0; i < num_recvrs; ++i) {
+ key = (char*) (recvrs + i);
+ if (stdhash_is_end(orig_membs, stdhash_find(orig_membs, &find, &key))) { /* a recvr wasn't */
+ DEBUG(std_stkfprintf(stderr, 0, "Illegal recvr '%s' not in ref '%s'\n", key, grp));
+ break;
+ }
+ }
+ } else { /* allow subgroupcast to private group */
+ for (i = 0; i < num_recvrs; ++i) { /* require ALL "receivers" to be same as grp */
+ key = (char*) (recvrs + i);
+ if (strncmp(grp, key, MAX_GROUP_NAME) != 0) {
+ DEBUG(std_stkfprintf(stderr, 0, "Illegal recvr '%s' not in ref '%s'\n", key, grp));
+ break;
+ }
+ }
+ }
+ fix_scat = 1; /* subgroup msgs have reference group appended to msg */
+ strncpy(curr_ptr, grp, MAX_GROUP_NAME);
+ curr_ptr += MAX_GROUP_NAME;
+
+ /* actually change the scatter to reflect appendings */
+ senders_elem = scat->elements[scat->num_elements]; /* save user's element */
+ scat->elements[scat->num_elements].buf = copy_buf;
+ scat->elements[scat->num_elements].len = (int) (curr_ptr - copy_buf);
+ ++scat->num_elements;
+
+ if (i == num_recvrs) /* all recvrs were legal above */
+ ret = SP_multigroup_scat_multicast(mbox, serv_type, num_recvrs,
+ (const char(*)[MAX_GROUP_NAME]) recvrs,
+ mess_type, scat);
+ else
+ ret = ILLEGAL_RECEIVERS;
+ }
+ if (fix_scat) { /* I modified user's scatter: I need to restore it */
+ --scat->num_elements;
+ if (ret >= 0) { /* successful return: need to adjust err to ignore appendings */
+ if (ret >= scat->elements[scat->num_elements].len)
+ ret -= scat->elements[scat->num_elements].len;
+ else
+ stderr_abort("(%s, %d): mbox %d: serv 0x%X: group '%s': SP_multicast returned %d\n",
+ __FILE__, __LINE__, mbox, serv_type, grp, ret);
+ }
+ scat->elements[scat->num_elements] = senders_elem; /* restore modified elem */
+ }
+ }
+ unlock_conn(conn); /* UNLOCK CONNECTION */
+ } else {
+ DEBUG(std_stkfprintf(stderr, 0, "FL ILLEGAL_SESSION(%d)\n", mbox));
+ ret = ILLEGAL_SESSION;
+ }
+ if (ret == CONNECTION_CLOSED || ret == ILLEGAL_SESSION)
+ FL_disconnect(mbox);
+
+ DEBUG(std_stkfprintf(stderr, -1, "FL_int_scat_multicast: ret %d, mbox %d\n", ret, mbox));
+ return ret;
+}
+
+/* try to read in a msg from the SP layer into m and make it presentable to the user
+ returns non-zero if user's parameters shouldn't be processed, but, instead returned
+*/
+static int FL_int_receive(gc_recv_mess *m) {
+ int alloced_groups = 0; /* boolean - did I alloc groups buffers? */
+ int alloced_buffer = 0; /* boolean - did I alloc a message buffer? */
+ int max_groups; /* max_groups to be passed to receive */
+ int orig_max_groups = m->max_groups; /* user's max_groups */
+ char (*groups)[MAX_GROUP_NAME] = m->groups; /* groups to be passed to receive */
+ scatter *scat = m->scat_mess; /* scatter to be passed to receive */
+ int success = 0; /* boolean - did it succeed? */
+
+ /* here I reserve space in groups for the destination group if it is a SUBGROUP_CAST msg */
+ if (m->max_groups != 0)
+ max_groups = m->max_groups - 1;
+ else
+ max_groups = 0;
+
+ /* don't use DROP_RECV semantics: flush needs to see entire data of msg: then we can drop */
+ *m->serv_type = (m->orig_serv_type & ~DROP_RECV);
+
+ DEBUG(std_stkfprintf(stderr, 1, "FL_int_receive: mbox %d, serv 0x%X, max groups %d, "
+ "grps %p, scat %p, scat_cap %ld\n", m->mbox, *m->serv_type, max_groups,
+ groups, scat, scat_capacity(scat)));
+
+ /* assert that there aren't any alloc'ed receive buffers, the user's
+ max_groups was legal and m doesn't already have a msg */
+ assert(m->num_new_grps == 0 && m->new_msg.num_elements == 0 &&
+ orig_max_groups >= 0 && !m->delivered);
+
+ /* perform the first receive */
+ m->ret = SP_scat_receive(m->mbox, m->serv_type, m->sender, max_groups, m->num_groups,
+ groups, m->mess_type, m->endian_mismatch, scat);
+
+ /* check for buffer errors: should be if buffers too short, I didn't use DROP_RECV */
+ if (m->ret == GROUPS_TOO_SHORT || m->ret == BUFFER_TOO_SHORT) {
+ DEBUG(std_stkfprintf(stderr, 0, "Buffer error after initial recv: ret %d, num_groups %d, "
+ "endian %d\n", m->ret, *m->num_groups, *m->endian_mismatch));
+
+ /* if the groups buffer was too short and the message was a SUBGROUP_CAST then the groups
+ buffer didn't have room for the destination group too: so decrement num_groups to reflect */
+ if (*m->num_groups < 0 && Is_subgroup_mess(*m->serv_type)) {
+ DEBUG(std_stkfprintf(stderr, 0, "GROUPS error, subgroup_mess: decrementing num_groups\n"));
+ --*m->num_groups;
+ }
+
+ /* if the user requested DROP_RECV, or if the user's groups was exactly big enough (because
+ I shrunk max_groups above) and there was no msg buffer error (reported by endian_mismatch)
+ or I can't tell if there was a buffer error because 3.12 didn't report that properly then
+ allocate the necessary buffers and re-receive - else return buffer error to user
+ */
+ if ((m->orig_serv_type & DROP_RECV) != 0 ||
+ (-*m->num_groups <= orig_max_groups &&
+ (*m->endian_mismatch == 0 || FL_SP_version() == (float) 3.12))) {
+ DEBUG(std_stkfprintf(stderr, 0, "Either DROP_RECV or might be able to re-receive!\n"));
+
+ /* user's groups array actually was too small to contain the groups, so allocate some */
+ if (-*m->num_groups > orig_max_groups) { /* implies *m->num_groups is negative */
+ size_t byte_size;
+
+ max_groups = -*m->num_groups; /* set max_groups to be big enough */
+ byte_size = max_groups * MAX_GROUP_NAME; /* calculate necessary memory size */
+
+ DEBUG(std_stkfprintf(stderr, 0, "User's GROUPS buff was actually TOO SMALL %d < %d\n",
+ orig_max_groups, -*m->num_groups));
+
+ alloced_groups = 1; /* record that I alloc'ed group buffers */
+ m->num_new_grps = max_groups; /* record size of alloc'ed group buffers */
+
+ /* actually allocate a new groups array to be used in re-receive */
+ if ((groups = m->new_grps = (char(*)[MAX_GROUP_NAME]) malloc(byte_size)) == 0)
+ stderr_abort("(%s, %d): malloc(%u)\n", __FILE__, __LINE__, byte_size);
+ } else {
+ DEBUG(std_stkfprintf(stderr, 0, "User's GROUPS buff was big enough %d >= %d\n",
+ orig_max_groups, -*m->num_groups));
+ max_groups = orig_max_groups; /* groups will fit into user's original buffers */
+ }
+
+ /* now check and see if the user's msg buffer was too small: necessary size reported in
+ endian_mismatch in versions later than 3.12, 3.12 didn't do it but did do DROP_RECV */
+ if (*m->endian_mismatch < 0 || FL_SP_version() == (float) 3.12) {
+ DEBUG(std_stkfprintf(stderr, 0, "Endian mismatch reports msg buffer is too small\n"));
+ alloced_buffer = 1; /* record that I alloc'ed a mess buffer */
+ scat = &m->new_msg; /* point scat at my alloc'ed mess buffer */
+ scat->num_elements = 1;
+
+ /* figure out how big the mess buffer needs to be */
+ if (FL_SP_version() != (float) 3.12)
+ scat->elements[0].len = -*m->endian_mismatch; /* capacity of msg to be recvd */
+ else
+ scat->elements[0].len = 102400; /* endian_mismatch is buggy, use max msg size */
+
+ /* actually allocate the new mess buffer */
+ if ((scat->elements[0].buf = (char*) malloc(scat->elements[0].len)) == 0)
+ stderr_abort("(%s, %d): malloc(%d)\n", __FILE__, __LINE__, scat->elements[0].len);
+ } else
+ DEBUG(std_stkfprintf(stderr, 0, "No msg buffer error reported\n"));
+
+ /* try to receive again (shouldn't fail now) and don't use DROP_RECV again */
+ *m->serv_type = (m->orig_serv_type & ~DROP_RECV);
+ DEBUG(std_stkfprintf(stderr, 0, "Re-receive: mbox %d, serv 0x%X, max groups %d, grps %p, "
+ "scat %p, scat_cap %ld\n", m->mbox, *m->serv_type, max_groups,
+ groups, scat, scat_capacity(scat)));
+
+ /* perform the re-receive */
+ m->ret = SP_scat_receive(m->mbox, m->serv_type, m->sender, max_groups, m->num_groups,
+ groups, m->mess_type, m->endian_mismatch, scat);
+
+ /* if we had a buffer problem then code above or SP is buggy -> abort */
+ if (m->ret == GROUPS_TOO_SHORT || m->ret == BUFFER_TOO_SHORT)
+ stderr_abort("(%s, %d): mbox %d: buggy SP_recv DROP_RECV ret %d: "
+ "max_groups %d: num_groups %d: scat_cap %ld: endian_mismatch %d\n",
+ __FILE__, __LINE__, m->mbox, m->ret, max_groups, *m->num_groups,
+ scat_capacity(scat), *m->endian_mismatch);
+ }
+ }
+ /* receive or re-receive succeeded */
+ if (m->ret >= 0) {
+ int not_end = 1; /* boolean - did I not seek to end of msg yet? */
+ scatp pos, user_pos;
+ long err;
+
+ /* successful recv: debug print out what I got! */
+ DEBUG(std_stkfprintf(stderr, 0, "Successful recv: ret %d, serv 0x%X, sender '%s', "
+ "num_groups %d, mess_type %d, endian %d, Groups:\n", m->ret,
+ *m->serv_type, m->sender, *m->num_groups,
+ *m->mess_type, *m->endian_mismatch);
+ for (err = 0; err < *m->num_groups; ++err) {
+ std_stkfprintf(stderr, 0, "\t`%s'\n", groups[err]);
+ });
+
+ /* subgroup and flush messages have appended data that needs to be extracted and removed */
+ if (Is_regular_mess(*m->serv_type)) {
+ if (Is_subgroup_mess(*m->serv_type)) { /* subgroup msgs have destination group appended */
+ char *ref_grp;
+
+ assert(m->ret >= MAX_GROUP_NAME); /* ensure fully recvd */
+ m->ret -= MAX_GROUP_NAME; /* correct to ignore the appendage */
+
+ err = scatp_set(&pos, scat, m->ret, SEEK_SET); /* seek to copy position */
+ assert(err == 0);
+ not_end = 0; /* pos was seeked to new end of msg */
+
+ /* copy out reference group: put after last used element in groups: should fit */
+ ref_grp = (char*) (groups + *m->num_groups);
+ err = scatp_cpy1(ref_grp, &pos, MAX_GROUP_NAME);
+ assert(err == MAX_GROUP_NAME);
+ ++*m->num_groups; /* increment num_groups to include destination group */
+ DEBUG(std_stkfprintf(stderr, 0, "Subgroupcast mess: Ref group is '%s'!\n", ref_grp));
+ }
+
+ /* all reserved FLUSH msg types have a group_id attached on end of msg */
+ if (IS_ILLEGAL_SEND_MTYPE(*m->mess_type)) {
+ DEBUG(std_stkfprintf(stderr, 0, "Recvd an internal flush mess of type %s\n",
+ *m->mess_type == FLUSH_OK_MESS ? "FLUSH_OK_MESS" :
+ (*m->mess_type == FLUSH_RECV_MESS ? "FLUSH_RECV_MESS" :
+ (*m->mess_type == VULNERABLE_MESS ? "VULNERABLE_MESS" :
+ "UNKNOWN TYPE!"))));
+ assert(m->ret >= sizeof(group_id)); /* ensure fully recvd */
+ m->ret -= sizeof(group_id); /* correct to ignore the appendage */
+
+ if (not_end) {
+ err = scatp_set(&pos, scat, m->ret, SEEK_SET); /* seek to copy position */
+ assert(err == 0);
+ not_end = 0; /* seeked to end */
+ } else {
+ err = scatp_jbackward(&pos, sizeof(group_id)); /* move back from end */
+ assert(err == sizeof(group_id));
+ }
+ err = scatp_cpy1((char*) &m->dst_gid, &pos, sizeof(group_id)); /* copy out vid */
+ assert(err == sizeof(group_id));
+
+ /* endian correct the vid if necessary */
+ if (*m->endian_mismatch != 0) {
+ stdflip32(m->dst_gid.id);
+ stdflip32(m->dst_gid.id + 1);
+ stdflip32(m->dst_gid.id + 2);
+ }
+ DEBUG(std_stkfprintf(stderr, 0, "Dst gid is %d %d %d!\n", m->dst_gid.id[0],
+ m->dst_gid.id[1], m->dst_gid.id[2]));
+
+ /* VULNERABLE msgs also have the original user's msg type appended */
+ if (*m->mess_type == VULNERABLE_MESS) {
+ assert(m->ret >= sizeof(int16)); /* ensure fully recvd */
+ m->ret -= sizeof(int16); /* correct to ignore the appendage */
+
+ m->vulnerable = 1; /* mark m as vulnerable */
+
+ /* here should already be seeked to end */
+ err = scatp_jbackward(&pos, sizeof(int16)); /* move back from end again */
+ assert(err == sizeof(int16));
+ err = scatp_cpy1((char*) m->mess_type, &pos, sizeof(int16)); /* cpy out mess type */
+ assert(err == sizeof(int16));
+
+ /* endian correct the message type if necessary */
+ if (*m->endian_mismatch != 0)
+ stdflip16(m->mess_type);
+
+ DEBUG(std_stkfprintf(stderr, 0, "VULNERABLE: Orig msg type is %d!\n", *m->mess_type));
+ }
+ }
+ } else if (Is_reg_memb_mess(*m->serv_type)) {
+ /* if its a regular membership message read the group id out of the msg */
+ assert(m->ret >= sizeof(group_id) + sizeof(int) + MAX_GROUP_NAME); /* ensure received */
+ err = scatp_begin(&pos, scat);
+ assert(err == 0);
+ err = scatp_cpy1((char*) &m->dst_gid, &pos, sizeof(group_id)); /* copy out new gid */
+ assert(err == sizeof(group_id)); /* already endian corrected */
+ DEBUG(std_stkfprintf(stderr, 0, "Recvd a SP reg memb mess: New SP gid is %d %d %d\n",
+ m->dst_gid.id[0], m->dst_gid.id[1], m->dst_gid.id[2]));
+ }
+
+ /* if I alloc'ed a msg buffer -> user probably requested DROP_RECV, need to copy from
+ alloc'ed buffers into user's buffers and set m->ret to an appropriate value */
+ if (alloced_buffer) {
+ m->new_msg.elements[0].len = m->ret; /* record size of user msg */
+ err = scatp_begin(&user_pos, m->scat_mess);
+ assert(err == 0);
+ err = scatp_begin(&pos, &m->new_msg);
+ assert(err == 0);
+
+ /* perform copy from alloc'ed scat to user's scat */
+ err = scatp_cpy0(&user_pos, &pos, m->ret);
+ assert(err >= 0 && err <= m->ret); /* shouldn't detect illegal scat here */
+
+ /* possible that removing appended data made the user's msg buffer big enough */
+ /* this also catches if I didn't really need to allocate a msg buffer (SP 3.12 bug) */
+ if (err != m->ret) { /* didn't all fit into user's msg buff */
+ DEBUG(std_stkfprintf(stderr, 0, "User msg buffer still too short!\n"));
+
+ /* 3.12: if !DROP_RECV, then report negative msg size correctly in endian_mismatch */
+ if ((m->orig_serv_type & DROP_RECV) == 0 && FL_SP_version() == (float) 3.12) {
+ DEBUG(std_stkfprintf(stderr, 0, "Alloced msg buff: SP 3.12 DROP_RECV bug: error!\n"));
+ success = -1; /* non-zero success indicates an error that should be returned now! */
+ *m->endian_mismatch = -m->ret;
+ }
+ m->ret = BUFFER_TOO_SHORT;
+ } else /* their buffer was big enough! */
+ DEBUG(std_stkfprintf(stderr, 0, "Unnecessary msg buff alloc! No BUFFER_TOO_SHORT!\n"));
+ }
+
+ /* if I alloc'ed groups buffers then user's buffers were too small and requested DROP_RECV */
+ if (alloced_groups) {
+ DEBUG(std_stkfprintf(stderr, 0, "Alloced groups buff, copying over: GROUPS_TOO_SHORT\n"));
+ memcpy(m->groups, m->new_grps, m->max_groups * MAX_GROUP_NAME);
+
+ /* if it is a subgroup msg and there is space put the destination group at end of groups */
+ if (Is_subgroup_mess(*m->serv_type) && m->max_groups > 0)
+ memcpy(m->groups[m->max_groups - 1], m->new_grps[*m->num_groups - 1], MAX_GROUP_NAME);
+
+ *m->num_groups = -*m->num_groups; /* report too short */
+ m->ret = GROUPS_TOO_SHORT;
+ }
+ } else {
+ DEBUG(std_stkfprintf(stderr, 0, "Error from recv or re-recv that needs to be returned\n"));
+ success = -1; /* non-zero success indicates an error that should be given to user now! */
+ }
+ DEBUG(std_stkfprintf(stderr, -1, "SP_int_receive: success %d, ret %d, mbox %d, serv 0x%X, "
+ "mess_type %d\n", success, m->ret, m->mbox, *m->serv_type,
+ *m->mess_type));
+ return success;
+}
+
+/* state machine code */
+/* recv_and_handle: main director function: returns whether or not still have conn lock */
+static int recv_and_handle(fl_conn *conn, gc_recv_mess *m) {
+ fl_group *group;
+ int ret = 1; /* do I still have the conn lock? */
+
+ DEBUG(std_stkfprintf(stderr, 1, "recv_and_handle: mbox(%d, %p)\n", conn->mbox, conn));
+ release_conn_lock(conn); /* let go of conn lock so I don't block sends on this connection */
+
+ if (FL_int_receive(m) != 0) { /* if an immediate return recv error: return to user */
+ DEBUG(std_stkfprintf(stderr, 0, "immediate return from FL_int_receive %d\n", m->ret));
+ m->delivered = 1;
+ ret = 0;
+ } else if (!acquire_conn_lock(conn)) { /* couldn't acquire conn lock -> disconnecting */
+ DEBUG(std_stkfprintf(stderr, 0, "Couldn't reacquire conn lock -> disconnecting\n"));
+ m->ret = ILLEGAL_SESSION;
+ m->delivered = 1;
+ ret = 0;
+ } else if (Is_regular_mess(*m->serv_type)) { /* SP regular message */
+ char (*dst_grp)[MAX_GROUP_NAME]; /* group this message is intended for */
+ int num_groups;
+
+ /* for regular msgs dst group is the first group, for subgroup msgs it is the last group */
+ get_groups_info(m, &num_groups, &dst_grp);
+ if (Is_subgroup_mess(*m->serv_type))
+ dst_grp += num_groups - 1;
+
+ DEBUG(std_stkfprintf(stderr, 0, "A regular mess destined for group '%s'\n", dst_grp));
+ if ((group = get_group(conn, (char*) dst_grp)) != 0) { /* look up fl_group structure */
+ if (*m->mess_type == FLUSH_OK_MESS)
+ handle_recv_flush_ok(conn, group, m);
+ else if (*m->mess_type == FLUSH_RECV_MESS)
+ handle_recv_flush_recv(conn, group, m);
+ else
+ handle_recv_reg_mess(conn, group, m);
+ } else if (strncmp((char*) dst_grp, conn->private, MAX_GROUP_NAME) == 0) { /* my pgroup */
+ DEBUG(std_stkfprintf(stderr, 0, "Mess is addressed to my private group! Deliver!\n"));
+ deliver(conn, m, 0, 0); /* deliver msg immediately */
+ } else
+ stderr_msg("(%s, %d): WARNING: a regular msg for a group of which I'm not a member!\n"
+ "mbox %d: serv 0x%X: sender '%s': dst_grp '%s': mess_type %d\n", __FILE__,
+ __LINE__, m->mbox, *m->serv_type, m->sender, dst_grp, *m->mess_type);
+ } else if (Is_membership_mess(*m->serv_type)) { /* SP membership message */
+ group = get_group(conn, m->sender); /* look up fl_group structure: might not exist */
+
+ DEBUG(std_stkfprintf(stderr, 0, "A memb mess destined for group ('%s', %p)\n",
+ m->sender, group));
+ if (Is_reg_memb_mess(*m->serv_type))
+ handle_recv_reg_memb_mess(conn, group, m);
+ else if (Is_transition_mess(*m->serv_type))
+ handle_recv_trans_memb_mess(conn, group, m);
+ else if (Is_caused_leave_mess(*m->serv_type))
+ handle_recv_self_leave_memb_mess(conn, group, m);
+ else
+ stderr_abort("(%s, %d): mbox %d: recv_and_handle: unexpected membership 0x%X: "
+ "group '%s'\n", __FILE__, __LINE__, conn->mbox, *m->serv_type, m->sender);
+ } else
+ stderr_abort("(%s, %d): mbox %d: recv_and_handle: unexpected service 0x%X: "
+ "sender '%s'\n", __FILE__, __LINE__, conn->mbox, *m->serv_type, m->sender);
+
+ DEBUG(std_stkfprintf(stderr, -1, "recv_and_handle: ret %d (have_conn_lock)\n", (int) ret));
+ return ret;
+}
+
+static void handle_recv_flush_ok(fl_conn *conn, fl_group *group, gc_recv_mess *m) {
+ sp_memb_change *spc;
+ stdit hit;
+ stdit lit;
+
+ DEBUG(std_stkfprintf(stderr, 1, "handle_recv_flush_ok: mbox(%d, %p), group('%s', %p), %s\n",
+ conn->mbox, conn, group->group, group, state_str(group->vstate)));
+
+ /* see if I know about this flok's gid yet or not: if not create a sp_memb_change */
+ if (!stdhash_is_end(&group->pmemb_hash, stdhash_find(&group->pmemb_hash, &hit, &m->dst_gid))) {
+ spc = *(sp_memb_change**) stdhash_it_val(&hit);
+ DEBUG(std_stkfprintf(stderr, 0, "Recvd a flok for a known gid %d %d %d: spc %p\n",
+ m->dst_gid.id[0], m->dst_gid.id[1], m->dst_gid.id[2], spc));
+ assert(SP_equal_group_ids(m->dst_gid, spc->memb_info->gid));
+ } else {
+ spc = create_sp_memb_change(m->dst_gid);
+ DEBUG(std_stkfprintf(stderr, 0, "Recvd a flok for an UNKNWON gid %d %d %d: CREATED spc %p\n",
+ m->dst_gid.id[0], m->dst_gid.id[1], m->dst_gid.id[2], spc));
+ stdhash_insert(&group->pmemb_hash, 0, &m->dst_gid, &spc); /* put into pmemb_hash */
+ }
+ stdhash_insert(&spc->flok_senders, 0, m->sender, 0); /* record flok from sender */
+ DEBUG(std_stkfprintf(stderr, 0, "curr_change(%p) ?= spc(%p) now has %lu floks, needs %d\n",
+ group->curr_change, spc, stdhash_size(&spc->flok_senders),
+ spc->memb_mess_recvd ? spc->memb_info->orig_num_membs : -1));
+ switch (group->vstate) {
+ case AGREE:
+ assert(group->curr_change != 0 && group->curr_change->memb_mess_recvd);
+ if (spc == group->curr_change &&
+ stdhash_size(&spc->flok_senders) == spc->memb_info->orig_num_membs) {
+ DEBUG(std_stkfprintf(stderr, 0, "Recvd last needed flok for curr change! INSTALL!\n"));
+ /* Received all the floks necessary to install the next fl view! */
+ install_new_view(conn, group, m); /* deliver an appropriate fl membership msg */
+
+ DEBUG(std_stkfprintf(stderr, 0, "Going to state VERIFY!\n"));
+ group->vstate = VERIFY; /* update group state */
+ if (group->mstate == JOINING)
+ group->mstate = JOINED; /* has been installed in a group */
+
+ update_fl_view(group); /* update the view information for this group */
+
+ /* if no cascading membership: send a FLUSH_RECV, else deliver a FLUSH_REQ mess */
+ if (stddll_empty(&group->memb_queue)) {
+ int err;
+
+ DEBUG(std_stkfprintf(stderr, 0, "No cascading memberships -> sending FLUSH_RECV\n"));
+ err = SP_multicast(conn->mbox, FIFO_MESS, group->group, FLUSH_RECV_MESS,
+ sizeof(group_id), (char*) &group->fl_view->gid);
+
+ if (err == CONNECTION_CLOSED || err == ILLEGAL_SESSION) { /* immediate return error */
+ DEBUG(std_stkfprintf(stderr, 0, "SP_multicast failure %ld\n", err));
+ m->delivered = 1; /* return to user immediately */
+ m->ret = (int) err;
+ break;
+ } else if (err != sizeof(group_id))
+ stderr_abort("(%s, %d): SP_multicast unexpected return %d\n", __FILE__, __LINE__, err);
+ } else { /* buffer a FLUSH_REQ mess, update curr_change, etc. */
+ DEBUG(std_stkfprintf(stderr, 0, "Cascading Memberships(%lu) to handle!\n",
+ stddll_size(&group->memb_queue)));
+ handle_next_memb_change(conn, group, m);
+ }
+ /* if new members have already died (after they sent their floks): deliver TRANS */
+ if (stdhash_size(&group->fl_view->curr_membs) < group->fl_view->orig_num_membs) {
+ DEBUG(std_stkfprintf(stderr, 0, "New view members have already died! Deliver TRANS!\n"));
+ assert(!group->fl_view->in_trans_memb);
+ group->fl_view->in_trans_memb = 1;
+ deliver_trans_sig(conn, m, group->group); /* will buffer */
+ }
+ /* deliver any VULNERABLE messages that were pre-delivered */
+ stddll_begin(&group->mess_queue, &lit);
+ for (; !stddll_is_end(&group->mess_queue, &lit); stddll_it_next(&lit)) {
+ DEBUG(std_stkfprintf(stderr, 0, "Delivering a vulnerable SP pre-delivered msg!\n"));
+ deliver(conn, m, *(gc_buff_mess**) stddll_it_val(&lit), 1); /* will buffer */
+ }
+ stddll_clear(&group->mess_queue); /* empty postponed vulnerable queue */
+ } else
+ assert(!spc->memb_mess_recvd ||
+ stdhash_size(&spc->flok_senders) < spc->memb_info->orig_num_membs);
+ break;
+ case AUTHORIZE:
+ assert(group->curr_change != 0 && group->curr_change->memb_mess_recvd);
+ assert(!spc->memb_mess_recvd ||
+ stdhash_size(&spc->flok_senders) < spc->memb_info->orig_num_membs);
+ break;
+ case STEADY: case VERIFY:
+ assert(group->curr_change == 0);
+ assert(!spc->memb_mess_recvd ||
+ stdhash_size(&spc->flok_senders) < spc->memb_info->orig_num_membs);
+ break;
+ default: stderr_abort("(%s, %d): impossible vstate %d\n", __FILE__, __LINE__, group->vstate);
+ }
+ DEBUG(std_stkfprintf(stderr, -1, "handle_recv_flush_ok: mbox(%d, %p), group('%s', %p), %s\n",
+ conn->mbox, conn, group->group, group, state_str(group->vstate)));
+}
+
+static void handle_recv_flush_recv(fl_conn *conn, fl_group *group, gc_recv_mess *m) {
+ DEBUG(std_stkfprintf(stderr, 1, "handle_recv_flush_recv: mbox(%d, %p), group('%s', %p), %s\n",
+ conn->mbox, conn, group->group, group, state_str(group->vstate)));
+ switch (group->vstate) {
+ case VERIFY:
+ assert(group->curr_change == 0);
+ if (SP_equal_group_ids(m->dst_gid, group->fl_view->gid)) {
+ DEBUG(std_stkfprintf(stderr, 0, "Got a FLUSH_RECV for curr FL vid need %d flush_recvs!\n",
+ group->fl_view->orig_num_membs));
+ if (++group->flush_recvs == group->fl_view->orig_num_membs) {
+ DEBUG(std_stkfprintf(stderr, 0, "Got last FLUSH_RECV: going to state STEADY!\n"));
+ group->vstate = STEADY;
+ }
+ } else
+ DEBUG(std_stkfprintf(stderr, 0, "Ignoring FLUSH_RECV for vid %d %d %d while in VERIFY!\n",
+ m->dst_gid.id[0], m->dst_gid.id[1], m->dst_gid.id[2]));
+ break;
+ case AGREE:
+ assert(group->curr_change != 0 && group->curr_change->memb_mess_recvd);
+ if (SP_equal_group_ids(m->dst_gid, group->curr_change->memb_info->gid)) {
+ DEBUG(std_stkfprintf(stderr, 0, "Got a FLUSH_RECV for CURR CHANGE need %d flush_recvs!\n",
+ group->fl_view->orig_num_membs));
+ ++group->flush_recvs;
+ assert(group->flush_recvs < group->curr_change->memb_info->orig_num_membs);
+ } else
+ DEBUG(std_stkfprintf(stderr, 0, "Ignoring FLUSH_RECV for vid %d %d %d while in AGREE!\n",
+ m->dst_gid.id[0], m->dst_gid.id[1], m->dst_gid.id[2]));
+ break;
+ case STEADY:
+ assert(group->curr_change == 0);
+ DEBUG(std_stkfprintf(stderr, 0, "Ignoring FLUSH_RECV for vid %d %d %d while in STEADY!\n",
+ m->dst_gid.id[0], m->dst_gid.id[1], m->dst_gid.id[2]));
+ break;
+ case AUTHORIZE:
+ assert(group->curr_change != 0 && group->curr_change->memb_mess_recvd);
+ DEBUG(std_stkfprintf(stderr, 0, "Ignoring FLUSH_RECV for vid %d %d %d while in AUTHORIZE!\n",
+ m->dst_gid.id[0], m->dst_gid.id[1], m->dst_gid.id[2]));
+ break;
+ default: stderr_abort("(%s, %d): impossible vstate %d\n", __FILE__, __LINE__, group->vstate);
+ }
+ DEBUG(std_stkfprintf(stderr, -1, "handle_recv_flush_recv: mbox(%d, %p), group('%s', %p), %s\n",
+ conn->mbox, conn, group->group, group, state_str(group->vstate)));
+}
+
+static void handle_recv_reg_mess(fl_conn *conn, fl_group *group, gc_recv_mess *um) {
+ stdit hit;
+ gc_buff_mess *bm;
+
+ DEBUG(std_stkfprintf(stderr, 1, "handle_recv_reg_mess: mbox(%d, %p), group('%s', %p), %s\n",
+ conn->mbox, conn, group->group, group, state_str(group->vstate)));
+ switch (group->vstate) {
+ case AGREE:
+ assert(group->curr_change != 0 && group->curr_change->memb_mess_recvd);
+ /* if vulnerable message meant for the next membership from a member of the next one */
+ if (um->vulnerable) {
+ DEBUG(std_stkfprintf(stderr, 0, "Recvd a vulnerable msg!\n"));
+ if (SP_equal_group_ids(um->dst_gid, group->curr_change->memb_info->gid)) {
+ DEBUG(std_stkfprintf(stderr, 0, "Addressed to my next FL vid!\n"));
+ if (!stdhash_is_end(&group->curr_change->memb_info->curr_membs, stdhash_find(&group->curr_change->memb_info->curr_membs,
+ &hit, &um->sender))) {
+ DEBUG(std_stkfprintf(stderr, 0, "Recvd a vuln msg that needs to be POSTPONED!\n"));
+ if ((bm = (gc_buff_mess*) malloc(sizeof(gc_buff_mess))) == 0)
+ stderr_abort("(%s, %d): malloc(%u)\n", __FILE__, __LINE__, sizeof(gc_buff_mess));
+
+ userm_to_buffm(bm, um);
+ stddll_push_back(&group->mess_queue, &bm);
+ break;
+ } else
+ stderr_abort("'%s' Not in my next FL curr membs!\n", um->sender);
+ } else
+ DEBUG(std_stkfprintf(stderr, 0, "Vuln msg not to next FL vid: should ignore below\n"));
+ } /* ELSE FALL THROUGH TO NORMAL CASES, BELOW (other case statements)!!!! */
+ case STEADY: case AUTHORIZE: case VERIFY:
+ /* if the message is from a current flush member then deliver it */
+ if (!stdhash_is_end(&group->fl_view->curr_membs, stdhash_find(&group->fl_view->curr_membs, &hit, &um->sender))) {
+ DEBUG(std_stkfprintf(stderr, 0, "Deliver reg mess from group memb '%s'\n", um->sender));
+ deliver(conn, um, 0, 0);
+ } else
+ DEBUG(std_stkfprintf(stderr, 0, "Ignore reg mess from non group memb '%s'\n", um->sender));
+ break;
+ default: stderr_abort("(%s, %d): impossible vstate %d\n", __FILE__, __LINE__, group->vstate);
+ }
+ DEBUG(std_stkfprintf(stderr, -1, "handle_recv_reg_mess: mbox(%d, %p), group('%s', %p), %s\n",
+ conn->mbox, conn, group->group, group, state_str(group->vstate)));
+}
+
+static void
+handle_recv_reg_memb_mess(fl_conn *conn, fl_group *group, gc_recv_mess *m) {
+ sp_memb_change *spc, *new_curr_change;
+ stdhash leavers; /* leavers<char*, nil>: members that left since last SP memb */
+ stdit hit, flit;
+ stdit lit;
+ char (*groups)[MAX_GROUP_NAME];
+ int num_groups;
+ int fl_memb_shrunk = 0;
+
+ DEBUG(std_stkfprintf(stderr, 1, "handle_recv_reg_memb_mess: mbox(%d, %p), group('%s', %p), %s"
+ ", gid %d %d %d\n", conn->mbox, conn, m->sender, group,
+ group != 0 ? state_str(group->vstate) : "NO STATE", m->dst_gid.id[0],
+ m->dst_gid.id[1], m->dst_gid.id[2]));
+
+ if (group == 0) { /* no fl_group struct for this group yet -> create one and put in conn */
+ group = add_group(conn, m->sender);
+ DEBUG(std_stkfprintf(stderr, 0, "No group structure yet, CREATED group %p!\n", group));
+ }
+ /* if a sp_memb_change for this gid doesn't exist yet: create one and put in pmemb */
+ if (stdhash_is_end(&group->pmemb_hash, stdhash_find(&group->pmemb_hash, &hit, &m->dst_gid))) {
+ spc = create_sp_memb_change(m->dst_gid);
+ DEBUG(std_stkfprintf(stderr, 0, "Unknown gid: CREATED spc %p\n", spc));
+ stdhash_insert(&group->pmemb_hash, 0, &m->dst_gid, &spc);
+ } else {
+ spc = *(sp_memb_change**) stdhash_it_val(&hit);
+ DEBUG(std_stkfprintf(stderr, 0, "KNOWN gid: got from pmemb_hash %p\n", spc));
+ }
+ get_groups_info(m, &num_groups, &groups); /* fill in rest of data for spc */
+ assert(!spc->memb_mess_recvd);
+ spc->memb_mess_recvd = 1; /* I got the memb mess for this gid */
+ spc->delta_member = determine_leavers(&leavers, group->sp_view, m); /* who left from last SP */
+ fill_view(spc->memb_info, *m->serv_type, num_groups, groups, *m->mess_type); /* spc view */
+
+ /* update the current fl_view: shrink the fl membership by removing any leavers */
+ for (stdhash_begin(&leavers, &hit); !stdhash_is_end(&leavers, &hit); stdhash_it_next(&hit)) {
+ if (!stdhash_is_end(&group->fl_view->curr_membs, stdhash_find(&group->fl_view->curr_membs,
+ &flit, stdhash_it_key(&hit)))) {
+ DEBUG(std_stkfprintf(stderr, 0, "FL member '%s' left in memb event, remove from fl view\n",
+ *(char**) stdhash_it_key(&hit)));
+ fl_memb_shrunk = 1; /* remember the membership shrunk */
+ stdhash_erase(&group->fl_view->curr_membs, &flit); /* remove from fl_view->curr_membs */
+ }
+ }
+ /* remove any memberships that this SP memb change invalidated */
+ age_and_invalidate_pmembs(group); /* age unknown spc's and remove them if too old */
+ new_curr_change = collapse_memberships(group, &leavers, spc); /* call before pushing spc!! */
+ stdhash_destruct(&leavers); /* after collapse members, leavers values could be invalid */
+
+ /* push spc on to end of memb queue and point sp view at spc's view */
+ /* spc must be pushed on to end of memb_queue after collapse memberships due to leave prob */
+ stddll_push_back(&group->memb_queue, &spc); /* push this SP memb change on to the memb queue */
+ group->sp_view = spc->memb_info; /* sp_view refers to most recent SP view */
+
+ /* if I wasn't working on a SP memb change or I just invalidated it -> update curr_change */
+ /* this will deliver a FLUSH_REQ message if I wasn't already working on a change */
+ if (group->curr_change != new_curr_change) {
+ DEBUG(std_stkfprintf(stderr, 0, "New SP memb change to work on now: %p -> %p!!\n",
+ group->curr_change, new_curr_change));
+
+ /* drop any pre-delivered messages as they must have come from partioned members */
+ for (stddll_begin(&group->mess_queue, &lit); !stddll_is_end(&group->mess_queue, &lit); stddll_it_next(&lit)) {
+ DEBUG(std_stkfprintf(stderr, 0, "Destroying pre-delivered msg from leaver member\n"));
+ free_gc_buff_mess(*(gc_buff_mess**) stddll_it_val(&lit));
+ }
+ stddll_clear(&group->mess_queue);
+ handle_next_memb_change(conn, group, m); /* possibly deliver a FLUSH_REQ */
+ }
+ /* if the group shrunk and I haven't delivered a trans yet -> deliver a trans signal */
+ if (!group->fl_view->in_trans_memb && fl_memb_shrunk) {
+ DEBUG(std_stkfprintf(stderr, 0, "FL memb shrunk, haven't delivered TRANS: deliver TRANS\n"));
+ group->fl_view->in_trans_memb = 1;
+ deliver_trans_sig(conn, m, group->group);
+ }
+ DEBUG(std_stkfprintf(stderr, -1, "handle_recv_reg_memb mess mbox(%d, %p), group('%s', %p), "
+ "%s\n", conn->mbox, conn, group->group, group, state_str(group->vstate)));
+}
+
+static void handle_next_memb_change(fl_conn *conn, fl_group *group, gc_recv_mess *m) {
+ stdit lit;
+ int err;
+
+ DEBUG(std_stkfprintf(stderr, 1, "handle_next_memb_change: mbox(%d, %p), group('%s', %p), "
+ "%s\n", conn->mbox, conn, group->group, group, state_str(group->vstate)));
+
+ assert(!stddll_empty(&group->memb_queue));
+ /* special case for JOINING: don't have a curr change yet! */
+ assert((group->vstate == STEADY || group->vstate == VERIFY) ?
+ group->curr_change == 0 : group->mstate == JOINING || group->curr_change != 0);
+
+ group->curr_change = *(sp_memb_change**) stddll_it_val(stddll_begin(&group->memb_queue, &lit));
+ assert(group->curr_change != 0 && group->curr_change->memb_mess_recvd);
+ group->flush_recvs = 0;
+
+ switch (group->vstate) {
+ case STEADY: case VERIFY:
+ DEBUG(std_stkfprintf(stderr, 0, "Going to state AUTHORIZE!\n"));
+ group->vstate = AUTHORIZE;
+ deliver_flush_req(conn, m, group->group);
+ break;
+ case AUTHORIZE: /* already waiting for a response from user */
+ break;
+ case AGREE:
+ DEBUG(std_stkfprintf(stderr, 0, "Going to state AUTHORIZE!\n"));
+ group->vstate = AUTHORIZE;
+ DEBUG(std_stkfprintf(stderr, 0, "AGREE: already auth'ed memb, send flok to new gid!\n"));
+ if ((err = FL_int_flush(conn, group)) != 0) { /* unrecoverable error */
+ DEBUG(std_stkfprintf(stderr, 0, "FL_int flush error: return to user\n"));
+ m->ret = err;
+ m->delivered = 1;
+ }
+ break;
+ default: stderr_abort("(%s, %d): impossible vstate %d\n", __FILE__, __LINE__, group->vstate);
+ }
+ DEBUG(std_stkfprintf(stderr, -1, "handle_next_memb_change: mbox(%d, %p), group('%s', %p), "
+ "%s\n", conn->mbox, conn, group->group, group, state_str(group->vstate)));
+}
+
+static void
+handle_recv_trans_memb_mess(fl_conn *conn, fl_group *group, gc_recv_mess *m) {
+ DEBUG(std_stkfprintf(stderr, 1, "handle_recv_trans_memb_mess: mbox(%d, %p), group('%s', %p), "
+ "%s\n", conn->mbox, conn, group->group, group, state_str(group->vstate)));
+ group->sp_view->in_trans_memb = 1;
+ if (!group->fl_view->in_trans_memb) {
+ DEBUG(std_stkfprintf(stderr, 0, "Got a TRANS for first time in FL view: deliver TRANS\n"));
+ group->fl_view->in_trans_memb = 1;
+ deliver_trans_sig(conn, m, group->group);
+ }
+ DEBUG(std_stkfprintf(stderr, -1, "handle_recv_trans_memb_mess: mbox(%d, %p), group('%s', %p), "
+ "%s\n", conn->mbox, conn, group->group, group, state_str(group->vstate)));
+}
+
+static void
+handle_recv_self_leave_memb_mess(fl_conn *conn, fl_group *group, gc_recv_mess *m){
+ DEBUG(std_stkfprintf(stderr, 1, "handle_recv_self_leave_memb_mess: mbox(%d, %p), "
+ "group('%s', %p), %s\n", conn->mbox, conn, group->group, group,
+ state_str(group->vstate)));
+ remove_group(conn, group);
+ deliver(conn, m, 0, 0);
+ DEBUG(std_stkfprintf(stderr, -1, "handle_recv_self_leave_memb_mess: mbox(%d, %p)\n",
+ conn->mbox, conn));
+}
+
+/* install the current view I was trying to finish */
+static void install_new_view(fl_conn *conn, const fl_group *group, gc_recv_mess *m) {
+ view *new_fl_view = group->curr_change->memb_info, *last_fl_view = group->fl_view;
+ int not_network = !Is_caused_network_mess(new_fl_view->memb_type);
+ char (*curr_vs)[MAX_GROUP_NAME];
+ gc_buff_mess *memb_mess;
+ size_t size, byte_size;
+
+ DEBUG(std_stkfprintf(stderr, 1, "install_new_view: mbox(%d, %p), group('%s', %p), %s\n",
+ conn->mbox, conn, group->group, group, state_str(group->vstate)));
+
+ /* I create a gc_buff_mess and then deliver it to the user for to simplify msg construction */
+ if ((memb_mess = (gc_buff_mess*) malloc(sizeof(gc_buff_mess))) == 0)
+ stderr_abort("(%s, %d): malloc(%u)\n", sizeof(gc_buff_mess));
+
+ memb_mess->mbox = conn->mbox; /* set mbox, service type, group, and num_groups */
+ memb_mess->serv_type = new_fl_view->memb_type;
+ assert(Is_reg_memb_mess(memb_mess->serv_type));
+ strncpy(memb_mess->sender, group->group, MAX_GROUP_NAME);
+ memb_mess->num_groups = new_fl_view->orig_num_membs;
+
+ byte_size = memb_mess->num_groups * MAX_GROUP_NAME; /* fill in the groups array */
+ if ((memb_mess->groups = (char(*)[MAX_GROUP_NAME]) malloc(byte_size)) == 0)
+ stderr_abort("(%s, %d): malloc(%u)\n", byte_size);
+ memcpy(memb_mess->groups, new_fl_view->membs_names, byte_size);
+
+ memb_mess->mess_type = new_fl_view->my_index; /* set mess_type and endian_mismatch */
+ memb_mess->endian_mismatch = 0;
+
+ if (not_network) { /* set the mess_len (return val) */
+ size = 1;
+ memb_mess->mess_len = sizeof(group_id) + sizeof(int) + MAX_GROUP_NAME;
+ } else {
+ size = stdhash_size(&last_fl_view->curr_membs);
+ byte_size = size * MAX_GROUP_NAME;
+ memb_mess->mess_len = sizeof(group_id) + sizeof(int) + byte_size;
+ }
+ if ((memb_mess->mess = (char*) malloc(memb_mess->mess_len)) == 0) /* allocate body of msg */
+ stderr_abort("(%s, %d): malloc(%u)\n", byte_size);
+
+ memcpy(memb_mess->mess, &new_fl_view->gid, sizeof(group_id)); /* fill in body: gid, num_vs */
+ memcpy(memb_mess->mess + sizeof(group_id), &size, sizeof(int));
+ curr_vs = (char(*)[MAX_GROUP_NAME]) (memb_mess->mess + sizeof(group_id) + sizeof(int));
+
+ /* fill in body of msg: vs_membs */
+ if (not_network)
+ memcpy(curr_vs, group->curr_change->delta_member, MAX_GROUP_NAME);
+ else {
+ char (*old_grps)[MAX_GROUP_NAME] = last_fl_view->membs_names;
+ char (*old_grps_end)[MAX_GROUP_NAME] = old_grps + last_fl_view->orig_num_membs;
+ stdit hit;
+
+ /* Step through the last fl_view's membs_names checking to see if each name is still */
+ /* in the membs (vs) set. This ensures the vs set is deterministically ordered. */
+ for (; old_grps != old_grps_end; ++old_grps)
+ if (!stdhash_is_end(&last_fl_view->curr_membs, stdhash_find(&last_fl_view->curr_membs, &hit, &old_grps)))
+ memcpy(curr_vs++, old_grps, MAX_GROUP_NAME);
+ }
+ deliver(conn, m, memb_mess, 1);
+ DEBUG(std_stkfprintf(stderr, -1, "install_new_view: mbox(%d, %p), group('%s', %p), %s\n",
+ conn->mbox, conn, group->group, group, state_str(group->vstate)));
+}
+
+static void update_fl_view(fl_group *group) {
+ stdit hit;
+ stdit lit;
+
+ DEBUG(std_stkfprintf(stderr, 1, "update_fl_view: group('%s', %p), %s, old gid %d %d %d\n",
+ group->group, group, state_str(group->vstate), group->fl_view->gid.id[0],
+ group->fl_view->gid.id[1], group->fl_view->gid.id[2]));
+ free_view(group->fl_view); /* delete old fl_view */
+ group->fl_view = group->curr_change->memb_info; /* steal curr_change's view */
+ group->curr_change->memb_info = 0; /* don't let fl_view be free'd below */
+
+ stddll_begin(&group->memb_queue, &lit);
+ assert(!stddll_is_end(&group->memb_queue, &lit) &&
+ group->curr_change == *(sp_memb_change**) stddll_it_val(&lit));
+ stddll_erase(&group->memb_queue, &lit); /* remove curr_change from head of memb_queue */
+ assert(stddll_is_end(&group->memb_queue, &lit) ||
+ group->curr_change != *(sp_memb_change**) stddll_it_val(&lit));
+
+ stdhash_find(&group->pmemb_hash, &hit, &group->fl_view->gid);
+ assert(!stdhash_is_end(&group->pmemb_hash, &hit) &&
+ group->curr_change == *(sp_memb_change**) stdhash_it_val(&hit));
+ stdhash_erase(&group->pmemb_hash, &hit); /* remove curr_change from pmemb_hash */
+ assert(stdhash_is_end(&group->pmemb_hash, stdhash_find(&group->pmemb_hash, &hit, &group->fl_view->gid)));
+
+ free_sp_memb_change(group->curr_change); /* reclaim non-stolen parts */
+ group->curr_change = 0;
+ DEBUG(std_stkfprintf(stderr, -1, "update_fl_view: group('%s', %p), %s, new gid %d %d %d\n",
+ group->group, group, state_str(group->vstate), group->fl_view->gid.id[0],
+ group->fl_view->gid.id[1], group->fl_view->gid.id[2]));
+}
+
+static char*
+determine_leavers(stdhash *leavers, view *last_sp_view, gc_recv_mess *m) {
+ char delta[MAX_GROUP_NAME], *delta_ptr = delta, *ret = 0;
+ int num_vs, i, *num_vs_ptr = &num_vs;
+ scatter *scat;
+ scatp pos;
+ long err;
+
+ DEBUG(std_stkfprintf(stderr, 1, "determine_leavers: old gid %d %d %d -> new gid %d %d %d\n",
+ last_sp_view->gid.id[0], last_sp_view->gid.id[1],
+ last_sp_view->gid.id[2], m->dst_gid.id[0], m->dst_gid.id[1],
+ m->dst_gid.id[2]));
+
+ get_scat_info(m, &i, &scat); /* i is a dummy variable here */
+ if (!Is_caused_network_mess(*m->serv_type)) { /* join, leave, or disconnection */
+ DEBUG(std_stkfprintf(stderr, 0, "Not caused by network! "));
+ stdhash_construct(leavers, sizeof(char*), 0,
+ group_name_ptr_cmp, group_name_ptr_hashcode, 0);
+
+ err = scatp_set(&pos, scat, sizeof(group_id) + sizeof(int), SEEK_SET);
+ assert(err == 0);
+ err = scatp_cpy1(delta, &pos, MAX_GROUP_NAME); /* read out joiner/leaver/disconnector */
+ assert(err == MAX_GROUP_NAME);
+ delta[MAX_GROUP_NAME - 1] = 0; /* ensure null termination */
+
+ if ((ret = (char*) malloc(MAX_GROUP_NAME)) == 0)
+ stderr_abort("(%s, %d): malloc(%d)\n", __FILE__, __LINE__, MAX_GROUP_NAME);
+ strncpy(ret, delta, MAX_GROUP_NAME); /* strndup delta member for return */
+
+ DEBUG(std_stkfprintf(stderr, 0, "delta memb: '%s'\n", delta));
+ if (!Is_caused_join_mess(*m->serv_type)) { /* insert leaver into leavers */
+ DEBUG(std_stkfprintf(stderr, 0, "Wasn't a JOIN: adding delta to leavers\n"));
+ stdhash_insert(leavers, 0, &ret, 0);
+ }
+ } else {
+ DEBUG(std_stkfprintf(stderr, 0, "Caused by network! "));
+ stdhash_copy_construct(leavers, &last_sp_view->orig_membs); /* copy last SP membership */
+
+ err = scatp_set(&pos, scat, sizeof(group_id), SEEK_SET);
+ assert(err == 0);
+ err = scatp_adv_cpy1((char**) &num_vs_ptr, &pos, sizeof(int), 0, 1); /* read out num_vs */
+ assert(err == sizeof(int));
+
+ DEBUG(std_stkfprintf(stderr, 0, "Num_vs %d\n", num_vs));
+ for (i = 0; i < num_vs; ++i) { /* remove members that came with me: leaves who left */
+ err = scatp_adv_cpy1(&delta_ptr, &pos, MAX_GROUP_NAME, 0, 1);
+ assert(err == MAX_GROUP_NAME);
+ delta[MAX_GROUP_NAME - 1] = 0; /* ensure null termination */
+ err = stdhash_size(leavers);
+ stdhash_erase_key(leavers, &delta_ptr);
+ assert(err == stdhash_size(leavers) + 1);
+ }
+ }
+ DEBUG(std_stkfprintf(stderr, -1, "determine_leavers: delta('%s', %p), leavers size is %lu\n",
+ ret != 0 ? ret : "", ret, stdhash_size(leavers)));
+ return ret; /* strndup'ed delta member or null */
+}
+
+static void age_and_invalidate_pmembs(fl_group *group) {
+ sp_memb_change *spc;
+ stdit hit;
+
+ DEBUG(std_stkfprintf(stderr, 1, "age_and_invalidate_pmembs: group('%s', %p), %s\n",
+ group->group, group, state_str(group->vstate)));
+
+ stdhash_begin(&group->pmemb_hash, &hit);
+ for (; !stdhash_is_end(&group->pmemb_hash, &hit); stdhash_it_next(&hit)) {
+ spc = *(sp_memb_change**) stdhash_it_val(&hit);
+ if (!spc->memb_mess_recvd && ++spc->memb_change_age > FLOK_AGE_LIMIT) {
+ DEBUG(std_stkfprintf(stderr, 0, "destroy too old spc %p, %d %d %d\n", spc,
+ spc->memb_info->gid.id[0], spc->memb_info->gid.id[1],
+ spc->memb_info->gid.id[2]));
+ stdhash_erase(&group->pmemb_hash, &hit);
+ free_sp_memb_change(spc);
+ break; /* only one memb can become too old per SP memb event */
+ }
+ }
+ DEBUG(std_stkfprintf(stderr, -1, "age_and_invalidate_pmembs: group('%s', %p), %s\n",
+ group->group, group, state_str(group->vstate)));
+}
+
+static sp_memb_change*
+collapse_memberships(fl_group *group, stdhash *leavers, sp_memb_change *c) {
+ stdit leavers_it, spc_it, flok_it, pmemb_it;
+ int invalidating = 0;
+ sp_memb_change *spc;
+ stdit lit;
+ char *leaver;
+
+ DEBUG(std_stkfprintf(stderr, 1, "collapse_memberships: group('%s', %p), %s, num_leavers %lu\n",
+ group->group, group, state_str(group->vstate), stdhash_size(leavers)));
+ if (!stddll_empty(&group->memb_queue) && !stdhash_empty(leavers)) {
+ for (stddll_begin(&group->memb_queue, &lit); !stddll_is_end(&group->memb_queue, &lit); ) { /* loop SP membs */
+ spc = *(sp_memb_change**) stddll_it_val(&lit);
+
+ stdhash_begin(leavers, &leavers_it); /* loop over leavers */
+ for (; !stdhash_is_end(leavers, &leavers_it); stdhash_it_next(&leavers_it)) {
+ leaver = *(char**) stdhash_it_key(&leavers_it);
+
+ /* see if a leaver is a member of this SP memb change */
+ if (!stdhash_is_end(&spc->memb_info->curr_membs, stdhash_find(&spc->memb_info->curr_membs, &spc_it, &leaver))) {
+ DEBUG(std_stkfprintf(stderr, 0, "leaver '%s' is in a pending membership!\n", leaver));
+ /* if leaver hasn't sent his flok yet, then invalidate this sp_memb_change: NO & !!! */
+ if (stdhash_is_end(&spc->flok_senders, stdhash_find(&spc->flok_senders, &flok_it, leaver))) {
+ DEBUG(std_stkfprintf(stderr, 0, "leaver didn't send flok: collapsing membship!\n"));
+ invalidating = 1;
+ stdhash_find(&group->pmemb_hash, &pmemb_it, &spc->memb_info->gid);
+ assert(!stdhash_is_end(&group->pmemb_hash, &pmemb_it));
+ stdhash_erase(&group->pmemb_hash, &pmemb_it); /* removes spc from pmemb hash */
+ stddll_erase(&group->memb_queue, &lit); /* removes spc from memb_queue and advances lit */
+ free_sp_memb_change(spc);
+ break; /* break out of inner loop: move on to next spc */
+ } else /* remove leaver from &spc->memb_info->curr_membs because he left */
+ stdhash_erase(&spc->memb_info->curr_membs, &spc_it);
+ }
+ }
+ if (stdhash_is_end(leavers, &leavers_it)) { /* didn't break out of loop due to an invalidation */
+ if (invalidating) { /* did invalidate some spc's previous to this one */
+ DEBUG(std_stkfprintf(stderr, 0, "collapsing invalidates here: CAUSED_BY_NETWORK\n"));
+ invalidating = 0;
+ spc->memb_info->memb_type = REG_MEMB_MESS | CAUSED_BY_NETWORK; /* collapse them here */
+ }
+ stddll_it_next(&lit); /* advance lit to next sp_memb_change */
+ }
+ }
+ if (invalidating) { /* invalidated last few on the memb queue: collapse them into c */
+ DEBUG(std_stkfprintf(stderr, 0, "collapsing invalidates into current spc\n"));
+ c->memb_info->memb_type = REG_MEMB_MESS | CAUSED_BY_NETWORK;
+ }
+ }
+ spc = (stddll_empty(&group->memb_queue) ? c :
+ *(sp_memb_change**) stddll_it_val(stddll_begin(&group->memb_queue, &lit)));
+
+ DEBUG(std_stkfprintf(stderr, -1, "collapse_memberships: curr change %p, group('%s', %p), %s\n",
+ spc, group->group, group, state_str(group->vstate)));
+ return spc;
+}
+
+static void get_groups_info(const gc_recv_mess *m, int *num_groups,
+ char (**groups)[MAX_GROUP_NAME]) {
+ if (m->num_new_grps == 0) {
+ *num_groups = *m->num_groups;
+ *groups = m->groups;
+ } else {
+ *num_groups = m->num_new_grps;
+ *groups = m->new_grps;
+ }
+}
+
+static void get_scat_info(const gc_recv_mess *m, int *mess_len,
+ scatter **scat_mess) {
+ if (m->new_msg.num_elements == 0) {
+ *mess_len = m->ret;
+ *scat_mess = (scatter*) m->scat_mess;
+ } else {
+ *mess_len = m->new_msg.elements[0].len;
+ *scat_mess = (scatter*) &m->new_msg;
+ }
+}
+
+/* is a group name a private group name or not? */
+static int is_private_group(const char group[MAX_GROUP_NAME]) {
+ const char *end = group + MAX_GROUP_NAME;
+
+ while (group != end && *group != 0 && *group != '#')
+ ++group;
+
+ return group != end && *group != 0;
+}
+
+/* check if a msg wrt a particular group is vulnerable or not */
+static int is_vulnerable_mess(const fl_group *group, service serv_type) {
+ /* need to include AUTHORIZE state because of coming here directly from the VERIFY state */
+ return ((group->vstate == AUTHORIZE || group->vstate == VERIFY) &&
+ (serv_type & (UNRELIABLE_MESS | RELIABLE_MESS | FIFO_MESS)) != 0);
+}
+
+static const char *state_str(fl_view_state state) {
+ switch (state) {
+ case STEADY: return "STEADY";
+ case AUTHORIZE: return "AUTHORIZE";
+ case AGREE: return "AGREE";
+ case VERIFY: return "VERIFY";
+ default: return "UNKOWN STATE!!!\n";
+ }
+}
+
+static float FL_SP_version(void) {
+#ifdef SPREAD_VERSION
+ int major, minor, patch, divBy;
+
+ SP_version(&major, &minor, &patch);
+
+ if (minor < 100)
+ divBy = 100;
+ else
+ divBy = 1000;
+
+ return major + (float) minor / divBy;
+#else
+ return SP_version();
+#endif
+}
+
+/* hashing fcns for group names */
+static stdbool group_name_cmp(const void *str1, const void *str2) {
+ return strncmp((const char*) str1, (const char*) str2, MAX_GROUP_NAME);
+}
+
+static size_t group_name_hashcode(const void *str) {
+ const stduint8 *key = (const stduint8*) str, *end = key + MAX_GROUP_NAME;
+ size_t ret = 0;
+
+ while (key != end && *key != 0)
+ ret = ret * 33 + *key++;
+
+ return ret;
+}
+
+static stdbool group_name_ptr_cmp(const void *strptr1, const void *strptr2) {
+ return strncmp(*(const char**) strptr1, *(const char**) strptr2, MAX_GROUP_NAME);
+}
+
+static size_t group_name_ptr_hashcode(const void *strptr) {
+ return group_name_hashcode(*(const void**) strptr);
+}
+
+static stdbool group_id_cmp(const void *gid1, const void *gid2) {
+ return memcmp(gid1, gid2, sizeof(group_id));
+}
+
+static size_t group_id_hashcode(const void *gid) {
+ group_id *gid2 = (group_id*) gid;
+
+ return (gid2->id[0] ^ gid2->id[1] ^ gid2->id[2]);
+}
+
Added: trunk/flush/fl_p.h
===================================================================
--- trunk/flush/fl_p.h 2005-07-28 14:50:56 UTC (rev 252)
+++ trunk/flush/fl_p.h 2005-07-31 10:51:40 UTC (rev 253)
@@ -0,0 +1,336 @@
+/*
+ * The contents of this file are subject to the FLUSH SPREAD Non-Commercial
+ * License, Version 1.0 (the ``License''); you may not use this file except
+ * in compliance with the License. You may obtain a copy of the License at:
+ *
+ * http://www.cnds.jhu.edu/research/group/flush_spread/FLUSH_LICENSE
+ *
+ * or in the file ``FLUSH_LICENSE'' found in the root of this distribution.
+ *
+ * Software distributed under the License is distributed on an AS IS basis,
+ * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+ * for the specific language governing rights and limitations under the
+ * License.
+ *
+ * The Original Software is:
+ * The Flush Spread Library
+ *
+ * The Initial Developers of the Original Software are:
+ * Yair Amir, John Schultz and Jonathan Stanton
+ *
+ * All Rights Reserved.
+ *
+ */
+
+#ifndef flush_p_h_2000_04_24_14_16_31_jschultz_at_cnds_jhu_edu
+#define flush_p_h_2000_04_24_14_16_31_jschultz_at_cnds_jhu_edu
+
+/* In this library I assume that the stdutil library I am linking with
+ was compiled with the -D STD_USE_EXCEPTIONS flag. This allows me to
+ assume that all reasonable calls to stdutil fcns won't fail, and if
+ they do (due to a system problem, such as malloc failing) they abort
+ with an appropriate error message, line number, etc.
+*/
+#include <fl.h>
+#include <scatp.h>
+#include <stdutil/stddefines.h>
+#include <stdutil/stderror.h>
+#include <stdutil/stdthread.h>
+#include <stdutil/stddll.h>
+#include <stdutil/stdhash.h>
+
+#define FL_MAJOR_VERSION 1
+#define FL_MINOR_VERSION 0
+#define FL_PATCH_VERSION 3
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/* The type fl_view_state specifies the possible states a connection
+ can be in, in reference to a particular group's membership algorithm.
+
+ STEADY - normal operation, member of the group
+ AUTHORIZE - a SP lvl memb event has occured, user needs to flush this group
+ AGREE - user has flushed this group, collecting floks for next fl view
+ VERIFY - new flush view installed, protect against non-causal msgs
+*/
+typedef enum { STEADY, AUTHORIZE, AGREE, VERIFY } fl_view_state;
+
+/* The type member_join_state specifies the possible states a connection
+ can be in, in reference to being a member of a particular group. Only
+ JOINED connections are allowed to leave and send to a group.
+
+ JOINING - this connection is in the process of joining this group
+ JOINED - this connection has been installed in a fl view of this group
+ LEAVING - this connection is leaving this group
+*/
+typedef enum { JOINING, JOINED, LEAVING } member_join_state;
+
+/* A gc_buff_mess contains all of the information necessary to replicate a message. */
+/* This is the struct I use when I have to buffer messages. */
+typedef struct {
+ mailbox mbox;
+ service serv_type;
+ char sender[MAX_GROUP_NAME];
+ int num_groups;
+ char (*groups)[MAX_GROUP_NAME];
+ int16 mess_type;
+ int endian_mismatch;
+ int mess_len;
+ char *mess;
+ int total_size; /* used for reporting number of bytes buffered: set by deliver() */
+} gc_buff_mess;
+
+/* A gc_recv_mess contains references to user's parameters from a call to receive. */
+/* I use this struct to pass data around in the internal state machine code. */
+/* The struct member "delivered" indicates whether or not the user's parameters have been */
+/* filled with legal return values and is currently returnable to the user or not. */
+typedef struct {
+ mailbox mbox;
+ service orig_serv_type; /* user's original service type request to receive */
+ service *serv_type;
+ char *sender;
+ int max_groups;
+ int *num_groups;
+ char (*groups)[MAX_GROUP_NAME];
+ int16 *mess_type;
+ int *endian_mismatch;
+ scatter *scat_mess;
+ int ret; /* return value from receive call */
+
+ int delivered; /* does this recv_mess already contain a delivered msg? */
+ int vulnerable; /* is this msg vulnerable? */
+ group_id dst_gid; /* the view id this message is addressed to */
+
+ int num_new_grps; /* potentially alloc'ed buffers for DROP_RECV msgs */
+ char (*new_grps)[MAX_GROUP_NAME];
+ scatter new_msg;
+} gc_recv_mess;
+
+/* This struct contains information that specifies a group membership view. */
+typedef struct {
+ group_id gid;
+ service memb_type;
+ int in_trans_memb; /* has a transitional msg been delivered in this view? */
+ int orig_num_membs;
+ char (*membs_names)[MAX_GROUP_NAME];
+ stdhash orig_membs, curr_membs; /* <char*, nil>, ptrs into membs_names */
+ int16 my_index; /* index of this connection in membs_names */
+} view;
+
+/* This struct contains information on a spread level membership
+ change for a particular group that has not yet been fully handled
+ by the flush layer. This structure is used by the fl memb protocol
+ while it attempts to install a view. These structures are built in
+ two different scenarios: (1) a FLUSH_OK or (2) a spread membership
+ message is received for a previously unseen/unknown view id.
+
+ There are 2 states that this structure can be in. These two states
+ correspond to whether or not a spread membership message for the
+ particular view id has been received yet or not. Which state the
+ structure is in is indicated by the struct member memb_mess_recvd.
+
+ The struct member memb_change_age indicates how many spread
+ membership messages have been received in this group since this
+ structure was built. If this value exceeds the value FLOK_AGE_LIMIT
+ and !memb_mess_recvd, then I take this as an indication that the
+ spread membership message that this structure is waiting for was
+ already delivered in a view of which I wasn't a member. In that
+ case, I will never receive that membership message and I can
+ "safely" discard this struct, as that membership change was already
+ handled by other members (this assumption is not 100% correct as
+ FIFO messages can jump before an unbounded number of memberships;
+ however, I feel that the probability of a FIFO message jumping
+ before FLOK_AGE_LIMIT memberships is vanishingly small).
+
+ If !memb_mess_recvd then the only fields that contain meaningful data
+ are memb_change_age, flok_senders, and memb_info->gid.
+
+ If memb_mess_recvd then delta_member contains either (1) a
+ malloc'ed copy of the delta member (if the membership change was
+ caused by join/leave/disconnect) or (2) null. All of the info in
+ memb_info is appropriately filled in when in this state.
+
+ The struct member flok_senders contains by value (ie - alloc'ed by
+ flok_senders) members' names who have flok'ed to this view_id.
+*/
+typedef struct {
+ view *memb_info;
+ int memb_mess_recvd;
+ int memb_change_age;
+ char *delta_member;
+ stdhash flok_senders; /* <char[MAX_GROUP_NAME], nil> */
+ group_id gid; /* temporary: erase me */
+} sp_memb_change;
+
+/* This struct contains all of the group level context for a connection in
+ reference to a particular group.
+
+ group_name - the name of the group with which this information is associated
+ mstate - current join state this member is in
+ vstate - current fl memb protocol state this member is in
+ curr_change - points to current sp_memb_change fl is trying to resolve (head of memb_queue)
+ sp_view - my most recently installed spread membership view (initially just me)
+ fl_view - my most recently installed flush membership view (initially just me)
+ flush_recvs - a count of how many flush_recvs have been recvd for fl_view
+ mess_queue - a queue of gc_buff_mess*s to be delivered later, in order
+ memb_queue - a queue of sp_memb_change*s to be handled in order
+ pmemb_hash - sp_memb_change*s that are being handled currently
+
+ sp_view either points at fl_view or at the most recent pending SP
+ memb event's view in memb_queue (see handle_recv_reg_memb_mess)
+
+ fl_view steals curr_change's view when curr_change is installed (see update_fl_view)
+*/
+typedef struct {
+ char group[MAX_GROUP_NAME];
+
+ member_join_state mstate;
+ fl_view_state vstate;
+
+ sp_memb_change *curr_change;
+
+ view *sp_view;
+ view *fl_view;
+ int flush_recvs;
+
+ stddll mess_queue; /* <gc_buff_mess*> */
+ stddll memb_queue; /* <sp_memb_change*> */
+ stdhash pmemb_hash; /* <group_id, sp_memb_change*> */
+} fl_group;
+
+/* This struct contains all of the connection level context for a connection */
+typedef struct {
+ stdmutex reserve_lock; /* protects reservations, disconnecting; destroy_cond's mutex */
+ size_t reservations; /* count of reservations on this connection */
+ int disconnecting; /* is this connection being disconnected? */
+ stdcond destroy_cond; /* disconnector thread waits on this until reservations == 0 */
+
+ stdmutex recv_lock; /* used to serialize calls to FL_recv on a connection */
+ stdmutex conn_lock; /* lock for access to the connection's data, see below */
+
+ /* acquiring conn_lock allows a thread to examine everything below here */
+ mailbox mbox;
+ int priority;
+ int group_memb;
+ char daemon[MAX_GROUP_NAME];
+ char user[MAX_GROUP_NAME];
+ char private[MAX_GROUP_NAME];
+
+ /* acquiring conn_lock allows a thread to examine and modify everything below here */
+ stdhash groups; /* information on groups involved in: <char*, fl_group*> */
+ stddll mess_queue; /* messages that the user can read out: <gc_buff_mess*> */
+ int bytes_queued; /* number of bytes available in mess_queue, used by FL_poll */
+} fl_conn;
+
+/************************* Private Variables, Functions and Macros *****************************/
+
+/* leave in or take out debugging printf's and checks? */
+#ifdef NDEBUG
+# define DEBUG(statement)
+#else
+# define DEBUG(statement) statement
+#endif
+
+/* age limit for a sp_memb_change with no sp memb mess */
+#define FLOK_AGE_LIMIT 200
+
+#define IS_ILLEGAL_SEND_STYPE(serv_type) \
+(((serv_type) & (FLUSH_REQ_MESS | MEMBERSHIP_MESS | ENDIAN_RESERVED | RESERVED)) != 0)
+
+/* mess types reserved by flush layer indicating special flush messages */
+/* MAKE SURE THAT FL_MIN_LEGAL_MESS_TYPE (in fl.h) IS ONE MORE THAN THE HIGHEST RESERVED MESS TYPE!!! */
+#define FLUSH_OK_MESS ((int16) -32768)
+#define FLUSH_RECV_MESS ((int16) -32767)
+#define VULNERABLE_MESS ((int16) -32766)
+#define IS_ILLEGAL_SEND_MTYPE(mess_type) ((mess_type) < FL_MIN_LEGAL_MESS_TYPE)
+
+/* structure routines */
+static fl_group *create_fl_group(const char *conn_name, const char *group_name);
+static void free_fl_group(fl_group *group);
+
+static void free_gc_buff_mess(gc_buff_mess *msg);
+
+static sp_memb_change *create_sp_memb_change(group_id gid);
+static void free_sp_memb_change(sp_memb_change *change);
+
+static view *create_view(group_id gid);
+static void free_view(view *v);
+static void fill_view(view *v, service memb_type, int num_membs,
+ char (*membs)[MAX_GROUP_NAME], int16 index);
+
+/* interface for reserving/locking connections */
+static fl_conn *lock_conn(mailbox mbox); /* reserve and lock connection */
+static void unlock_conn(fl_conn *conn);
+static fl_conn *make_reservation(mailbox mbox); /* declare interest in connection */
+static void cancel_reservation(fl_conn *conn);
+static int acquire_conn_lock(fl_conn *conn); /* get rights to read/modify */
+static void release_conn_lock(fl_conn *release);
+static int acquire_recv_lock(fl_conn *conn); /* get rights to enter FL_recv */
+static void release_recv_lock(fl_conn *release);
+
+/* interface for groups for a connection */
+static fl_group *add_group(fl_conn *conn, const char *group);
+static void remove_group(fl_conn *conn, fl_group *group);
+static fl_group *get_group(fl_conn *conn, const char *grp);
+
+/* functions for trying to deliver/return msgs to the user */
+static gc_buff_mess *deliver(fl_conn *c, gc_recv_mess *um, gc_buff_mess *bm, int al);
+static gc_buff_mess *deliver_trans_sig(fl_conn *conn, gc_recv_mess *um, char *group);
+static gc_buff_mess *deliver_flush_req(fl_conn *conn, gc_recv_mess *um, char *group);
+
+/* converting back and forth between user's variables and buffered messages (dst, src) */
+static int buffm_to_userm(gc_recv_mess *um, const gc_buff_mess *bm);
+static void userm_to_buffm(gc_buff_mess *bm, const gc_recv_mess *um);
+
+/* complex "low level" internal fcns that deal with SP layer */
+static int FL_int_flush(fl_conn *conn, fl_group *group);
+static int FL_int_scat_multicast(mailbox m, service s, const char *g,
+ int nr, char r[][MAX_GROUP_NAME],
+ int16 mess_type, const scatter *scat);
+static int FL_int_receive(gc_recv_mess *m);
+
+/* state machine fcns: handler functions */
+static int recv_and_handle(fl_conn *c, gc_recv_mess *m);
+static void handle_recv_flush_ok(fl_conn *c, fl_group *g, gc_recv_mess *m);
+static void handle_recv_flush_recv(fl_conn *c, fl_group *g, gc_recv_mess *m);
+static void handle_recv_reg_mess(fl_conn *c, fl_group *g, gc_recv_mess *m);
+static void handle_recv_reg_memb_mess(fl_conn *c, fl_group *g, gc_recv_mess *m);
+static void handle_next_memb_change(fl_conn *c, fl_group *g, gc_recv_mess *m);
+static void handle_recv_trans_memb_mess(fl_conn *c, fl_group *g, gc_recv_mess *m);
+static void handle_recv_self_leave_memb_mess(fl_conn *c, fl_group *g, gc_recv_mess *m);
+
+/* helper functions for handler functions */
+static void install_new_view(fl_conn *c, const fl_group *g, gc_recv_mess *m);
+static void update_fl_view(fl_group *group);
+static char *determine_leavers(stdhash *leavers, view *last_sp_view, gc_recv_mess *m);
+static void age_and_invalidate_pmembs(fl_group *group);
+static sp_memb_change *collapse_memberships(fl_group *group, stdhash *leavers,
+ sp_memb_change *c);
+/* miscellaneous functions */
+/* where is full data for a msg (ie - groups or new_grps?) due to DROP_RECV considerations */
+static void get_groups_info(const gc_recv_mess *m, int *num_groups,
+ char (**groups)[MAX_GROUP_NAME]);
+static void get_scat_info(const gc_recv_mess *m, int *mess_len, scatter **scat_mess);
+
+static int is_private_group(const char *group);
+static int is_vulnerable_mess(const fl_group *group, service serv_type);
+
+static const char *state_str(fl_view_state state);
+
+static float FL_SP_version(void);
+
+/* group name hash fcns: used by stdhashs where the keys are group_names */
+static stdbool group_name_cmp(const void *str1, const void *str2);
+static size_t group_name_hashcode(const void *str);
+static stdbool group_name_ptr_cmp(const void *str_ptr1, const void *str_ptr2);
+static size_t group_name_ptr_hashcode(const void *str_ptr);
+static stdbool group_id_cmp(const void *gid1, const void *gid2);
+static size_t group_id_hashcode(const void *gid);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
Added: trunk/flush/fl_time_memb.c
===================================================================
--- trunk/flush/fl_time_memb.c 2005-07-28 14:50:56 UTC (rev 252)
+++ trunk/flush/fl_time_memb.c 2005-07-31 10:51:40 UTC (rev 253)
@@ -0,0 +1,261 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <math.h>
+#include <limits.h>
+#include <float.h>
+
+#include <sys/time.h>
+#include <sys/resource.h>
+#include <unistd.h>
+
+#include "fl.h"
+#include "stats.h"
+
+#define MY_MAX_NUM_GROUPS 1000
+#define MY_MAX_MESS_SIZE 102400
+
+enum { LOAD_MEMBER = 0, DELTA = 1, DIE_MESS = 2048 };
+
+mailbox mbox;
+service service_type;
+char sender[MAX_GROUP_NAME];
+int num_groups;
+char groups[MY_MAX_NUM_GROUPS][MAX_GROUP_NAME];
+int16 mess_type;
+int endian_mismatch;
+int mess_len;
+char mess[MY_MAX_MESS_SIZE];
+int more_messes;
+
+char daemon_name[MAX_GROUP_NAME] = "4803 at localhost";
+char user_name[MAX_GROUP_NAME] = "1";
+char priv_name[MAX_GROUP_NAME];
+char group_name[MAX_GROUP_NAME] = "test";
+
+char *exe = 0; /* executable's name */
+int user_type = -1; /* user type of application */
+int num_joins_leaves = 0; /* i: tmp var, num_joins_leaves: # of memberships to cause */
+int about_to_die = 0; /* boolean: am I dieing after next membership? */
+int num_members;
+
+int should_sleep = 1; /* should there be sleeps between memberships? Ususally yes. */
+int pretty_print = 1; /* should the output be verbose and labeled for human consumption? */
+
+/* the results of the scenarios timings */
+stats_results sp_join_stats, sp_leave_stats, sp_cmbo_stats;
+stats_results fl_join_stats, fl_leave_stats, fl_cmbo_stats;
+
+int i, err, num_joins;
+
+static int printUsage(FILE *outstream) {
+ return fprintf(outstream,
+ "Usage: %s\r\n"
+ "\t[-S <group size>] : # members in group (for stats; also login name)\r\n"
+ "\t[-s <address>] : spread daemon name - either port or port at machine\r\n"
+ "\t[-g <group name>] : group name to join\r\n"
+ "\t[-t <user type> <# joins/leaves>] : type of user "
+ "(LOAD_MEMBER = %d, DELTA = %d), # of join/leave events\r\n"
+ "\t[-f] : don't sleep between memberships (default is to sleep)\r\n"
+ "\t[-r] : print raw stats w/ no pretty headings\r\n",
+ exe, LOAD_MEMBER, DELTA);
+}
+
+static void usage(int argc, char **argv) {
+ for (++argv, --argc; argc > 0; ++argv, --argc) {
+ if (!strcmp(*argv, "-S") && --argc) {
+ strncpy(user_name, *++argv, MAX_GROUP_NAME);
+
+ } else if (!strcmp(*argv, "-s") && --argc) {
+ strncpy(daemon_name, *++argv, MAX_GROUP_NAME);
+
+ } else if (!strcmp(*argv, "-g") && --argc) {
+ strncpy(group_name, *++argv, MAX_GROUP_NAME);
+
+ } else if (!strcmp(*argv, "-t") && (argc -= 2) > 0) {
+ user_type = atoi(*++argv);
+ num_joins_leaves = atoi(*++argv);
+
+ } else if (!strcmp(*argv, "-f")) {
+ should_sleep = 0;
+
+ } else if (!strcmp(*argv, "-r")) {
+ pretty_print = 0;
+
+ } else {
+ fprintf(stderr, "Unknown cmd line param: %s\r\n", *argv);
+ exit(printUsage(stderr));
+ }
+ }
+ num_members = atoi(user_name);
+}
+
+int main(int argc, char **argv) {
+ double t;
+
+ exe = *argv;
+ usage(argc, argv);
+
+ if ((err = FL_connect(daemon_name, user_name, 0, &mbox, priv_name)) != ACCEPT_SESSION) {
+ fprintf(stderr, "FL_connect failure: ");
+ FL_error(err);
+ exit(1);
+ }
+ if (user_type == DELTA) {
+ double *fl_join_times = (double*) malloc(sizeof(double) * num_joins_leaves);
+ double *fl_leave_times = (double*) malloc(sizeof(double) * num_joins_leaves);
+ double *fl_cmbo_times = (double*) malloc(sizeof(double) * num_joins_leaves);
+
+ if (!fl_join_times || !fl_leave_times || !fl_cmbo_times) {
+ exit(fprintf(stderr, "Couldn't mallocate tracking arrays!\r\n"));
+ }
+
+ for (i = 0; i < num_joins_leaves; ++i) {
+ t = get_time_timeofday();
+
+ if ((err = FL_join(mbox, group_name)) < 0) {
+ fprintf(stderr, "FL_join failure: ");
+ FL_error(err);
+ exit(1);
+ }
+ do {
+ if ((mess_len = FL_receive(mbox, &service_type, sender, MY_MAX_NUM_GROUPS,
+ &num_groups, groups, &mess_type, &endian_mismatch,
+ MY_MAX_MESS_SIZE, mess, &more_messes)) < 0) {
+ fprintf(stderr, "FL_receive failure: ");
+ FL_error(mess_len);
+ exit(1);
+ }
+ } while (!Is_reg_memb_mess(service_type));
+
+ fl_join_times[i] = get_time_timeofday() - t;
+
+ if (should_sleep) {
+ /* sleep for 4 times how long the join membership took */
+ /* allows membership to stabilize */
+ usleep((unsigned long) (4 * fl_join_times[i] * 1000));
+ }
+
+ /* send a kill message if we are done */
+ if (i == num_joins_leaves - 1) {
+ if ((mess_len = FL_multicast(mbox, SAFE_MESS, group_name, DIE_MESS, 0, 0)) < 0) {
+ fprintf(stderr, "FL_multicast failure: ");
+ FL_error(err);
+ exit(1);
+ }
+ }
+
+ t = get_time_timeofday();
+
+ if ((err = FL_leave(mbox, group_name)) < 0) {
+ fprintf(stderr, "FL_leave failure: ");
+ FL_error(err);
+ exit(1);
+ }
+ do {
+ if ((mess_len = FL_receive(mbox, &service_type, sender, MY_MAX_NUM_GROUPS,
+ &num_groups, groups, &mess_type, &endian_mismatch,
+ MY_MAX_MESS_SIZE, mess, &more_messes)) < 0) {
+ fprintf(stderr, "FL_receive failure: ");
+ FL_error(mess_len);
+ exit(1);
+ }
+ } while (!Is_self_leave(service_type));
+
+ fl_leave_times[i] = get_time_timeofday() - t;
+ fl_cmbo_times[i] = fl_join_times[i] + fl_leave_times[i];
+
+ if (should_sleep) {
+ /* sleep for 4 times how long the join membership took */
+ /* allows membership to stabilize */
+ usleep((unsigned long) (4 * fl_join_times[i] * 1000));
+ }
+ }
+ /* compute timing statistics */
+ comp_stats(&fl_join_stats, fl_join_times, num_joins_leaves);
+ comp_stats(&fl_leave_stats, fl_leave_times, num_joins_leaves);
+ comp_stats(&fl_cmbo_stats, fl_cmbo_times, num_joins_leaves);
+
+ /* output timing statistics */
+ if (pretty_print) {
+ printf("Flush Membership Timings (includes Spread): Group Size: %d, # Joins/Leaves: %d\r\n\r\n",
+ num_members, num_joins_leaves);
+
+ printf("\tFlush Join: ind. total (90%%): %.6fms, ind. total: %.6fms\r\n",
+ sp_join_stats.total90, sp_join_stats.total);
+
+ printf("\tFlush Leave: ind. total (90%%): %.6fms, ind. total: %.6fms\r\n",
+ sp_leave_stats.total90, sp_leave_stats.total);
+
+ printf("\tFlush Join/Leave: ind. total (90%%): %.6fms, ind. total: %.6fms\r\n",
+ sp_cmbo_stats.total90, sp_cmbo_stats.total);
+
+ printf("\r\n\t\tGroup Size\t# Trials\tQ1\tMEDIAN\tQ3"
+ "\tMIN (5%%)\tMEAN90\tSTDDEV90\tMAX (95%%)"
+ "\tMIN\tMEAN\tSTDDEV\tMAX\r\n");
+ }
+ printf("\tFL_Join:\t%d\t%d\t%.6f\t%.6f\t%.6f\t%.6f\t%.6f\t%.6f\t%.6f\t%.6f\t%.6f\t%.6f\t%.6f\r\n",
+ num_members, num_joins_leaves, fl_join_stats.quart1, fl_join_stats.median,
+ fl_join_stats.quart3, fl_join_stats.min5, fl_join_stats.mean90,
+ fl_join_stats.stddev90, fl_join_stats.max95, fl_join_stats.min,
+ fl_join_stats.mean, fl_join_stats.stddev, fl_join_stats.max);
+
+ printf("\tFL_Leave:\t%d\t%d\t%.6f\t%.6f\t%.6f\t%.6f\t%.6f\t%.6f\t%.6f\t%.6f\t%.6f\t%.6f\t%.6f\r\n",
+ num_members, num_joins_leaves, fl_leave_stats.quart1, fl_leave_stats.median,
+ fl_leave_stats.quart3, fl_leave_stats.min5, fl_leave_stats.mean90,
+ fl_leave_stats.stddev90, fl_leave_stats.max95, fl_leave_stats.min,
+ fl_leave_stats.mean, fl_leave_stats.stddev, fl_leave_stats.max);
+
+ printf("\tFL_Join + FL_Leave\t%d\t%d\t%.6f\t%.6f\t%.6f\t%.6f\t%.6f\t%.6f\t%.6f\t%.6f\t%.6f\t%.6f\t%.6f\r\n",
+ num_members, num_joins_leaves, fl_cmbo_stats.quart1, fl_cmbo_stats.median,
+ fl_cmbo_stats.quart3, fl_cmbo_stats.min5, fl_cmbo_stats.mean90,
+ fl_cmbo_stats.stddev90, fl_cmbo_stats.max95, fl_cmbo_stats.min,
+ fl_cmbo_stats.mean, fl_cmbo_stats.stddev, fl_cmbo_stats.max);
+
+ } else if (user_type == LOAD_MEMBER) {
+ if ((err = FL_join(mbox, group_name)) < 0) {
+ fprintf(stderr, "FL_join failure: ");
+ FL_error(err);
+ exit(1);
+ }
+ while (1) {
+ if ((mess_len = FL_receive(mbox, &service_type, sender,
+ MY_MAX_NUM_GROUPS, &num_groups, groups,
+ &mess_type, &endian_mismatch,
+ MY_MAX_MESS_SIZE, mess, &more_messes)) < 0) {
+ fprintf(stderr, "FL_receive failure: ");
+ FL_error(mess_len);
+ exit(1);
+ }
+ if (Is_flush_req_mess(service_type)) {
+ if ((err = FL_flush(mbox, group_name)) < 0) {
+ fprintf(stderr, "FL_flush failure: ");
+ FL_error(err);
+ exit(1);
+ }
+ } else if (Is_reg_memb_mess(service_type)) {
+ if (Is_caused_leave_mess(service_type)) {
+ if (about_to_die) {
+ break;
+ }
+ } else if (!Is_caused_join_mess(service_type)) {
+ exit(fprintf(stderr, "Unexpected membership type: %d\n", service_type));
+ }
+ } else if (Is_safe_mess(service_type) && mess_type == DIE_MESS) {
+ about_to_die = 1;
+ }
+ }
+ printf("Success!\r\n");
+
+ } else {
+ fprintf(stderr, "Unknown user type: %d\n", user_type);
+ exit(printUsage(stderr));
+ }
+
+ if ((err = FL_disconnect(mbox)) < 0) {
+ fprintf(stderr, "FL_disconnect failure: ");
+ FL_error(err);
+ exit(1);
+ }
+ return 0;
+}
Added: trunk/flush/scatp.c
===================================================================
--- trunk/flush/scatp.c 2005-07-28 14:50:56 UTC (rev 252)
+++ trunk/flush/scatp.c 2005-07-31 10:51:40 UTC (rev 253)
@@ -0,0 +1,440 @@
+/*
+ * The contents of this file are subject to the FLUSH SPREAD Non-Commercial
+ * License, Version 1.0 (the ``License''); you may not use this file except
+ * in compliance with the License. You may obtain a copy of the License at:
+ *
+ * http://www.cnds.jhu.edu/research/group/flush_spread/FLUSH_LICENSE
+ *
+ * or in the file ``FLUSH_LICENSE'' found in the root of this distribution.
+ *
+ * Software distributed under the License is distributed on an AS IS basis,
+ * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+ * for the specific language governing rights and limitations under the
+ * License.
+ *
+ * The Original Software is:
+ * The Flush Spread Library
+ *
+ * The Initial Developers of the Original Software are:
+ * Yair Amir, John Schultz and Jonathan Stanton
+ *
+ * All Rights Reserved.
+ *
+ */
+
+#include <errno.h>
+#include <assert.h>
+#include <string.h>
+#include <scatp.h>
+
+long scat_capacity(const scatter *scat)
+{
+ const scat_element *curr = scat->elements, *end = scat->elements + scat->num_elements;
+ long ret = 0;
+
+ if (scat->num_elements < 0 || scat->num_elements > MAX_CLIENT_SCATTER_ELEMENTS)
+ return ILLEGAL_MESSAGE;
+
+ for (; curr != end; ++curr) {
+ if (curr->len < 0)
+ return ILLEGAL_MESSAGE;
+ ret += curr->len;
+ }
+ return ret;
+}
+
+int scatp_begin(scatp *pos, const scatter *scat)
+{
+ int i;
+
+ if (scat->num_elements < 0 || scat->num_elements > MAX_CLIENT_SCATTER_ELEMENTS)
+ return ILLEGAL_MESSAGE;
+
+ for (i = 0; i < scat->num_elements && scat->elements[i].len == 0; ++i);
+
+ if (i != scat->num_elements && scat->elements[i].len < 0)
+ return ILLEGAL_MESSAGE;
+
+ pos->scat = (scatter*) scat;
+ pos->elem_ind = i;
+ pos->buff_ind = 0;
+ return 0;
+}
+
+int scatp_end(scatp *pos, const scatter *scat)
+{
+ if (scat->num_elements < 0 || scat->num_elements > MAX_CLIENT_SCATTER_ELEMENTS)
+ return ILLEGAL_MESSAGE;
+
+ pos->scat = (scatter*) scat;
+ pos->elem_ind = scat->num_elements;
+ pos->buff_ind = 0;
+ return 0;
+}
+
+int scatp_set(scatp *pos, const scatter *scat, long offset, int whence)
+{
+ int err;
+
+ if (whence == SEEK_CUR) {
+ if ((err = scatp_begin(pos, scat)))
+ return err;
+ } else
+ pos->scat = (scatter*) scat;
+
+ return scatp_seek(pos, offset, whence);
+}
+
+int scatp_is_legal(const scatp *pos)
+{
+ const scatter *scat = pos->scat;
+
+ return (scat->num_elements >= 0 && scat->num_elements <= MAX_CLIENT_SCATTER_ELEMENTS &&
+ (scatp_is_end(pos) ||
+ (pos->elem_ind >= 0 && pos->elem_ind < scat->num_elements &&
+ pos->buff_ind >= 0 && pos->buff_ind < scat->elements[pos->elem_ind].len)));
+}
+
+int scatp_is_not_legal(const scatp *pos)
+{
+ const scatter *scat = pos->scat;
+
+ return (scat->num_elements < 0 || scat->num_elements > MAX_CLIENT_SCATTER_ELEMENTS ||
+ (!scatp_is_end(pos) &&
+ (pos->elem_ind < 0 || pos->elem_ind >= scat->num_elements ||
+ pos->buff_ind < 0 || pos->buff_ind >= scat->elements[pos->elem_ind].len)));
+}
+
+int scatp_is_end(const scatp *pos)
+{
+ if (pos->scat->num_elements < 0 || pos->scat->num_elements > MAX_CLIENT_SCATTER_ELEMENTS)
+ return ILLEGAL_MESSAGE;
+
+ return pos->elem_ind == pos->scat->num_elements && pos->buff_ind == 0;
+}
+
+int scatp_equals(const scatp *pos1, const scatp *pos2)
+{
+ if (scatp_is_not_legal(pos1) || scatp_is_not_legal(pos2))
+ return ILLEGAL_MESSAGE;
+
+ if (pos1->scat != pos2->scat)
+ return ILLEGAL_SERVICE;
+
+ return pos1->elem_ind == pos2->elem_ind && pos1->buff_ind == pos2->buff_ind;
+}
+
+long scatp_comp(const scatp *pos1, const scatp *pos2)
+{
+ const scat_element *curr, *end;
+ const scatter *scat = pos1->scat;
+ long ret;
+
+ if (scatp_is_not_legal(pos1) || scatp_is_not_legal(pos2))
+ return ILLEGAL_MESSAGE;
+
+ if (pos1->scat != pos2->scat)
+ return ILLEGAL_SERVICE;
+
+ if (pos1->elem_ind == pos2->elem_ind)
+ return pos1->buff_ind - pos2->buff_ind;
+
+ if (pos1->elem_ind < pos2->elem_ind) {
+ ret = pos1->buff_ind - scat->elements[pos1->elem_ind].len;
+ curr = scat->elements + pos1->elem_ind;
+ end = scat->elements + pos2->elem_ind;
+ while (++curr != end) {
+ if (curr->len < 0)
+ return ILLEGAL_MESSAGE;
+ ret -= curr->len;
+ }
+ } else {
+ ret = scat->elements[pos2->elem_ind].len - pos2->buff_ind;
+ curr = scat->elements + pos2->elem_ind;
+ end = scat->elements + pos1->elem_ind;
+ while (++curr != end) {
+ if (curr->len < 0)
+ return ILLEGAL_MESSAGE;
+ ret += curr->len;
+ }
+ }
+ return ret;
+}
+
+/* This fcn moves a scat_pos forward num_bytes bytes in a scatter. On
+ success, this fcn returns num_bytes and pos is modified
+ appropriately. Otherwise, if the jump was so big that it would have
+ jumped off the end of the scatter, the number of bytes that _could_
+ have been successfully jumped is returned, and pos is unchanged.
+*/
+
+long scatp_jforward(scatp *pos, long num_bytes)
+{
+ long elem_ind, skip_bytes, tmp;
+ const scatter *scat = pos->scat;
+
+ if (scatp_is_not_legal(pos))
+ return ILLEGAL_MESSAGE;
+
+ if (num_bytes < 0)
+ return ILLEGAL_SERVICE;
+
+ if (scatp_is_end(pos)) /* can't move forward from end */
+ return 0;
+
+ /* jump stays within current element buffer */
+ if ((tmp = scat->elements[pos->elem_ind].len - pos->buff_ind) > num_bytes) {
+ pos->buff_ind += num_bytes;
+ return num_bytes;
+ }
+ /* incorporate what was left in current element buffer into jump */
+ elem_ind = pos->elem_ind + 1;
+ skip_bytes = num_bytes - tmp; /* how many bytes left to skip */
+
+ for (; elem_ind < scat->num_elements; ++elem_ind) {
+ if (scat->elements[elem_ind].len < 0)
+ return ILLEGAL_MESSAGE;
+
+ /* use < 0 because it jumps over any zero length buffers */
+ if ((skip_bytes -= scat->elements[elem_ind].len) < 0) {
+ skip_bytes += scat->elements[elem_ind].len; /* restore to positive */
+ break;
+ }
+ }
+ /* jump forward jumped past end of scatter */
+ if (elem_ind == scat->num_elements && skip_bytes != 0)
+ return num_bytes - skip_bytes;
+
+ pos->elem_ind = elem_ind;
+ pos->buff_ind = skip_bytes;
+ return num_bytes;
+}
+
+/* same as scat_jforward, except it moves the position backwards in the scatter */
+
+long scatp_jbackward(scatp *pos, long num_bytes)
+{
+ long elem_ind, e_ind, skip_bytes;
+ const scatter *scat = pos->scat;
+
+ if (scatp_is_not_legal(pos))
+ return ILLEGAL_MESSAGE;
+
+ if (num_bytes < 0)
+ return ILLEGAL_SERVICE;
+
+ /* jump stays within current element buffer */
+ if (pos->buff_ind >= num_bytes) {
+ pos->buff_ind -= num_bytes;
+ return num_bytes;
+ }
+ /* incorporate what was left in current element buffer into jump */
+ elem_ind = pos->elem_ind;
+ skip_bytes = num_bytes - pos->buff_ind;
+
+ for (e_ind = pos->elem_ind - 1; e_ind >= 0; --e_ind) {
+ if (scat->elements[e_ind].len < 0)
+ return ILLEGAL_MESSAGE;
+
+ /* again we want to ignore any zero length buffers */
+ if (scat->elements[e_ind].len > 0) {
+ elem_ind = e_ind; /* elem_ind must reference a non-empty element buffer */
+ if ((skip_bytes -= scat->elements[e_ind].len) <= 0)
+ break;
+ }
+ }
+ if (e_ind < 0)
+ return num_bytes - skip_bytes;
+
+ pos->elem_ind = elem_ind;
+ pos->buff_ind = -skip_bytes; /* restore to positive */
+ return num_bytes;
+}
+
+int scatp_seek(scatp *pos, long offset, int whence)
+{
+ scatp set;
+ long err;
+
+ switch (whence) {
+ case SEEK_CUR:
+ set = *pos;
+ break;
+ case SEEK_SET:
+ if ((err = scatp_begin(&set, pos->scat)))
+ return (int) err;
+ break;
+ case SEEK_END:
+ if ((err = scatp_end(&set, pos->scat)))
+ return (int) err;
+ break;
+ default:
+ return EINVAL;
+ }
+ if (offset >= 0) {
+ if ((err = scatp_jforward(&set, offset)) != offset)
+ return err < 0 ? (int) err : -1;
+ } else {
+ offset = -offset;
+ if ((err = scatp_jbackward(&set, offset)) != offset)
+ return err < 0 ? (int) err : -1;
+ }
+ *pos = set;
+ return 0;
+}
+
+long scatp_cpy0(const scatp *dst, const scatp *src, long num_bytes)
+{
+ return scatp_adv_cpy0((scatp*) dst, (scatp*) src, num_bytes, 0, 0);
+}
+
+long scatp_cpy1(char *dst, const scatp *src, long num_bytes)
+{
+ scatter dscat;
+ scatp dscatp;
+ long err;
+
+ dscat.num_elements = 1;
+ dscat.elements[0].len = num_bytes;
+ dscat.elements[0].buf = dst;
+
+ err = scatp_begin(&dscatp, &dscat);
+ assert(err == 0);
+
+ return scatp_cpy0(&dscatp, src, num_bytes);
+}
+
+long scatp_cpy2(const scatp *dst, char *src, long num_bytes)
+{
+ scatter sscat;
+ scatp sscatp;
+ long err;
+
+ sscat.num_elements = 1;
+ sscat.elements[0].len = num_bytes;
+ sscat.elements[0].buf = src;
+
+ err = scatp_begin(&sscatp, &sscat);
+ assert(err == 0);
+
+ return scatp_cpy0(dst, &sscatp, num_bytes);
+}
+
+long scatp_adv_cpy0(scatp *dst, scatp *src, long num_bytes, int adv_dst, int adv_src)
+{
+ scatter *dscat = dst->scat, *sscat = src->scat;
+ long dst_elem, src_elem, bytes_left, copy_size, dst_left, src_left;
+ char *dst_curr, *dst_end, *src_curr, *src_end;
+
+ if (scatp_is_not_legal(dst) || scatp_is_not_legal(src)) {
+ printf("illegal scatp! dst: %d src: %d\n", scatp_is_not_legal(dst), scatp_is_not_legal(src));
+ return ILLEGAL_MESSAGE;
+ }
+ if (num_bytes < 0)
+ return ILLEGAL_SERVICE;
+
+ if (scatp_is_end(dst) || scatp_is_end(src))
+ return 0;
+
+ dst_elem = dst->elem_ind;
+ dst_curr = dscat->elements[dst->elem_ind].buf + dst->buff_ind;
+ dst_end = dscat->elements[dst->elem_ind].buf + dscat->elements[dst->elem_ind].len;
+
+ src_elem = src->elem_ind;
+ src_curr = sscat->elements[src->elem_ind].buf + src->buff_ind;
+ src_end = sscat->elements[src->elem_ind].buf + sscat->elements[src->elem_ind].len;
+
+ bytes_left = num_bytes;
+
+ while (dst_elem < dscat->num_elements && src_elem < sscat->num_elements && bytes_left) {
+ dst_left = dst_end - dst_curr;
+ src_left = src_end - src_curr;
+ copy_size = (dst_left < src_left) ? dst_left : src_left;
+ copy_size = (copy_size < bytes_left) ? copy_size : bytes_left;
+
+ if (copy_size < 0) { /* ensure no empty or negative copies */
+ printf("scatp_cpy: buffer size negative!\n");
+ return ILLEGAL_MESSAGE;
+ }
+
+ memcpy(dst_curr, src_curr, copy_size);
+ bytes_left -= copy_size;
+
+ if (copy_size != dst_left)
+ dst_curr += copy_size;
+ else {
+ while (++dst_elem < dscat->num_elements && dscat->elements[dst_elem].len == 0);
+ if (dst_elem < dscat->num_elements) {
+ dst_curr = dscat->elements[dst_elem].buf;
+ dst_end = dscat->elements[dst_elem].buf + dscat->elements[dst_elem].len;
+ }
+ }
+ if (copy_size != src_left)
+ src_curr += copy_size;
+ else {
+ while (++src_elem < sscat->num_elements && sscat->elements[src_elem].len == 0);
+ if (src_elem < sscat->num_elements) {
+ src_curr = sscat->elements[src_elem].buf;
+ src_end = sscat->elements[src_elem].buf + sscat->elements[src_elem].len;
+ }
+ }
+ }
+ if (bytes_left != 0) /* couldn't do the entire copy */
+ return num_bytes - bytes_left;
+
+ /* success! now, update the scatp's if requested to */
+ if (adv_dst) {
+ dst->elem_ind = dst_elem;
+ if (dst_elem != dscat->num_elements)
+ dst->buff_ind = dst_curr - dscat->elements[dst_elem].buf;
+ else
+ dst->buff_ind = 0;
+ }
+ if (adv_src) {
+ src->elem_ind = src_elem;
+ if (src_elem != sscat->num_elements)
+ src->buff_ind = src_curr - sscat->elements[src_elem].buf;
+ else
+ src->buff_ind = 0;
+ }
+ return num_bytes;
+}
+
+long scatp_adv_cpy1(char **dst, scatp *src, long num_bytes, int adv_dst, int adv_src)
+{
+ scatter dscat;
+ scatp dscatp;
+ long ret;
+
+ dscat.num_elements = 1;
+ dscat.elements[0].len = num_bytes;
+ dscat.elements[0].buf = *dst;
+
+ ret = scatp_begin(&dscatp, &dscat);
+ assert(ret == 0);
+
+ if ((ret = scatp_adv_cpy0(&dscatp, src, num_bytes, 0, adv_src)) == num_bytes && adv_dst)
+ *dst += num_bytes;
+
+ return ret;
+}
+
+long scatp_adv_cpy2(scatp *dst, char **src, long num_bytes, int adv_dst, int adv_src)
+{
+ scatter sscat;
+ scatp sscatp;
+ long ret;
+
+ sscat.num_elements = 1;
+ sscat.elements[0].len = num_bytes;
+ sscat.elements[0].buf = *src;
+
+ ret = scatp_begin(&sscatp, &sscat);
+ assert(ret == 0);
+
+ if ((ret = scatp_adv_cpy0(dst, &sscatp, num_bytes, adv_dst, 0)) == num_bytes && adv_src)
+ *src += num_bytes;
+
+ return ret;
+}
+
Added: trunk/flush/scatp.h
===================================================================
--- trunk/flush/scatp.h 2005-07-28 14:50:56 UTC (rev 252)
+++ trunk/flush/scatp.h 2005-07-31 10:51:40 UTC (rev 253)
@@ -0,0 +1,78 @@
+/*
+ * The contents of this file are subject to the FLUSH SPREAD Non-Commercial
+ * License, Version 1.0 (the ``License''); you may not use this file except
+ * in compliance with the License. You may obtain a copy of the License at:
+ *
+ * http://www.cnds.jhu.edu/research/group/flush_spread/FLUSH_LICENSE
+ *
+ * or in the file ``FLUSH_LICENSE'' found in the root of this distribution.
+ *
+ * Software distributed under the License is distributed on an AS IS basis,
+ * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+ * for the specific language governing rights and limitations under the
+ * License.
+ *
+ * The Original Software is:
+ * The Flush Spread Library
+ *
+ * The Initial Developers of the Original Software are:
+ * Yair Amir, John Schultz and Jonathan Stanton
+ *
+ * All Rights Reserved.
+ *
+ */
+
+#ifndef scatp_h_2000_05_21_19_14_47_jschultz_at_cnds_jhu_edu
+#define scatp_h_2000_05_21_19_14_47_jschultz_at_cnds_jhu_edu
+
+#include <sp.h>
+#include <stdio.h>
+
+/* don't try to initialize this structure by yourself, instead call
+ scatp_begin, scatp_set, or scatp_end -- there are special cases
+ where the initial values are not obvious
+*/
+
+typedef struct scatp /* represents a logical position within a scatter structure */
+{
+ scatter *scat;
+ long elem_ind; /* use signed because scatter uses signed on some archs */
+ long buff_ind;
+
+} scatp;
+
+long scat_capacity(const scatter *scat);
+
+/* initializers: need to call one to init a scatp properly */
+
+int scatp_begin(scatp *pos, const scatter *scat);
+int scatp_end(scatp *pos, const scatter *scat);
+int scatp_set(scatp *pos, const scatter *scat, long offset, int whence);
+
+/* some information about a scatp */
+
+int scatp_is_legal(const scatp *pos);
+int scatp_is_not_legal(const scatp *pos);
+int scatp_is_end(const scatp *pos);
+int scatp_equals(const scatp *pos1, const scatp *pos2);
+long scatp_comp(const scatp *pos1, const scatp *pos2);
+
+/* move current position of a scatp */
+
+long scatp_jforward(scatp *pos, long num_bytes);
+long scatp_jbackward(scatp *pos, long num_bytes);
+int scatp_seek(scatp *pos, long offset, int whence);
+
+/* copy from a src to a dst like memcpy does */
+
+long scatp_cpy0(const scatp *dst, const scatp *src, long num_bytes);
+long scatp_cpy1(char *dst, const scatp *src, long num_bytes);
+long scatp_cpy2(const scatp *dst, char *src, long num_bytes);
+
+/* copy from a src to a dst like memcpy does, and advance the positions if so requested */
+
+long scatp_adv_cpy0(scatp *dst, scatp *src, long num_bytes, int adv_dst, int adv_src);
+long scatp_adv_cpy1(char **dst, scatp *src, long num_bytes, int adv_dst, int adv_src);
+long scatp_adv_cpy2(scatp *dst, char **src, long num_bytes, int adv_dst, int adv_src);
+
+#endif
Added: trunk/flush/sp_time_memb.c
===================================================================
--- trunk/flush/sp_time_memb.c 2005-07-28 14:50:56 UTC (rev 252)
+++ trunk/flush/sp_time_memb.c 2005-07-31 10:51:40 UTC (rev 253)
@@ -0,0 +1,253 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <math.h>
+#include <limits.h>
+#include <float.h>
+
+#include <sys/time.h>
+#include <sys/resource.h>
+#include <unistd.h>
+
+#include "fl.h"
+#include "stats.h"
+
+#define MY_MAX_NUM_GROUPS 1000
+#define MY_MAX_MESS_SIZE 102400
+
+enum { LOAD_MEMBER = 0, DELTA = 1, DIE_MESS = 2048 };
+
+mailbox mbox;
+service service_type;
+char sender[MAX_GROUP_NAME];
+int num_groups;
+char groups[MY_MAX_NUM_GROUPS][MAX_GROUP_NAME];
+int16 mess_type;
+int endian_mismatch;
+int mess_len;
+char mess[MY_MAX_MESS_SIZE];
+int more_messes;
+
+char daemon_name[MAX_GROUP_NAME] = "4803 at localhost";
+char user_name[MAX_GROUP_NAME] = "1";
+char priv_name[MAX_GROUP_NAME];
+char group_name[MAX_GROUP_NAME] = "test";
+
+char *exe = 0; /* executable's name */
+int user_type = -1;
+int num_joins_leaves = 0; /* i: tmp var, num_joins_leaves: # of memberships to cause */
+int about_to_die = 0; /* boolean: am I dieing after next membership? */
+int num_members;
+
+int should_sleep = 1; /* should there be sleeps between memberships? ususally yes */
+int pretty_print = 1; /* should the output be verbose and human readable? */
+
+stats_results sp_join_stats, sp_leave_stats, sp_cmbo_stats;
+
+int i, err, num_joins;
+
+static int printUsage(FILE *outstream) {
+ return fprintf(outstream,
+ "Usage: %s\r\n"
+ "\t[-S <group size>] : group size (for stats; also login name, defaults to 1)\r\n"
+ "\t[-s <address>] : spread daemon name - either port or port at machine\r\n"
+ "\t[-g <group name>] : group name to join\r\n"
+ "\t[-t <user type> <# joins/leaves>] : type of user "
+ "(LOAD_MEMBER = %d, DELTA = %d) + # of join/leave events\r\n"
+ "\t[-f] : don't sleep between memberships (default is to sleep)\r\n"
+ "\t[-r] : print raw scores w/ no pretty headings\r\n",
+ exe, LOAD_MEMBER, DELTA);
+}
+
+static void usage(int argc, char **argv) {
+ for (++argv, --argc; argc > 0; ++argv, --argc) {
+ if (!strcmp(*argv, "-S") && --argc) {
+ strncpy(user_name, *++argv, MAX_GROUP_NAME);
+
+ } else if (!strcmp(*argv, "-s") && --argc) {
+ strncpy(daemon_name, *++argv, MAX_GROUP_NAME);
+
+ } else if (!strcmp(*argv, "-g") && --argc) {
+ strncpy(group_name, *++argv, MAX_GROUP_NAME);
+
+ } else if (!strcmp(*argv, "-t") && (argc -= 2) > 0) {
+ user_type = atoi(*++argv);
+ num_joins_leaves = atoi(*++argv);
+
+ } else if (!strcmp(*argv, "-f")) {
+ should_sleep = 0;
+
+ } else if (!strcmp(*argv, "-r")) {
+ pretty_print = 0;
+
+ } else {
+ fprintf(stderr, "Unknown cmd line param: %s\r\n", *argv);
+ exit(printUsage(stderr));
+ }
+ }
+ num_members = atoi(user_name);
+}
+
+int main(int argc, char **argv) {
+ exe = *argv;
+
+ usage(argc, argv);
+
+ if ((err = SP_connect(daemon_name, user_name, 0, 1, &mbox, priv_name)) != ACCEPT_SESSION) {
+ fprintf(stderr, "SP_connect failure: ");
+ SP_error(err);
+ exit(1);
+ }
+ if (user_type == DELTA) {
+ double *sp_join_times = (double*) malloc(sizeof(double) * num_joins_leaves);
+ double *sp_leave_times = (double*) malloc(sizeof(double) * num_joins_leaves);
+ double *sp_cmbo_times = (double*) malloc(sizeof(double) * num_joins_leaves);
+
+ if (!sp_join_times || !sp_leave_times || !sp_cmbo_times) {
+ exit(fprintf(stderr, "Couldn't mallocate tracking arrays!\r\n"));
+ }
+
+ for (i = 0; i < num_joins_leaves; ++i) {
+ double t = get_time_timeofday();
+
+ if ((err = SP_join(mbox, group_name)) < 0) {
+ fprintf(stderr, "SP_join failure: ");
+ SP_error(err);
+ exit(1);
+ }
+ do {
+ if ((mess_len = SP_receive(mbox, &service_type, sender, MY_MAX_NUM_GROUPS,
+ &num_groups, groups, &mess_type, &endian_mismatch,
+ MY_MAX_MESS_SIZE, mess)) < 0) {
+ fprintf(stderr, "SP_receive failure: ");
+ SP_error(mess_len);
+ exit(1);
+ }
+ } while (!Is_reg_memb_mess(service_type));
+
+ sp_join_times[i] = get_time_timeofday() - t;
+
+ if (should_sleep) {
+ /* sleep for 4 times how long the join membership took */
+ /* allows membership to stabilize */
+ usleep((unsigned long) (4 * sp_join_times[i] * 1000));
+ }
+
+ /* send a kill message if we are done */
+ if (i == num_joins_leaves - 1) {
+ if ((mess_len = SP_multicast(mbox, SAFE_MESS, group_name, DIE_MESS, 0, 0)) < 0) {
+ fprintf(stderr, "SP_multicast failure: ");
+ SP_error(err);
+ exit(1);
+ }
+ }
+ t = get_time_timeofday();
+
+ if ((err = SP_leave(mbox, group_name)) < 0) {
+ fprintf(stderr, "SP_leave failure: ");
+ SP_error(err);
+ exit(1);
+ }
+ do {
+ if ((mess_len = SP_receive(mbox, &service_type, sender, MY_MAX_NUM_GROUPS,
+ &num_groups, groups, &mess_type, &endian_mismatch,
+ MY_MAX_MESS_SIZE, mess)) < 0) {
+ fprintf(stderr, "SP_receive failure: ");
+ SP_error(mess_len);
+ exit(1);
+ }
+ } while (!Is_self_leave(service_type));
+
+ sp_leave_times[i] = get_time_timeofday() - t;
+ sp_cmbo_times[i] = sp_join_times[i] + sp_leave_times[i];
+
+ if (should_sleep) {
+ /* sleep for 4 times how long the join membership took */
+ /* allows membership to stabilize */
+ usleep((unsigned long) (4 * sp_join_times[i] * 1000));
+ }
+ }
+ /* compute statistics */
+ comp_stats(&sp_join_stats, sp_join_times, num_joins_leaves);
+ comp_stats(&sp_leave_stats, sp_leave_times, num_joins_leaves);
+ comp_stats(&sp_cmbo_stats, sp_cmbo_times, num_joins_leaves);
+
+ /* output statistics */
+ if (pretty_print) {
+ printf("Spread Membership Timings: Group Size: %d, # Joins/Leaves: %d\r\n\r\n",
+ num_members, num_joins_leaves);
+
+ printf("\tSpread Join: ind. total (90%%): %.6fms, ind. total: %.6fms\r\n",
+ sp_join_stats.total90, sp_join_stats.total);
+
+ printf("\tSpread Leave: ind. total (90%%): %.6fms, ind. total: %.6fms\r\n",
+ sp_leave_stats.total90, sp_leave_stats.total);
+
+ printf("\tSpread Join/Leave: ind. total (90%%): %.6fms, ind. total: %.6fms\r\n",
+ sp_cmbo_stats.total90, sp_cmbo_stats.total);
+
+ printf("\r\n\t\tGroup Size\t# Trials\tQ1\tMEDIAN\tQ3"
+ "\tMIN (5%%)\tMEAN90\tSTDDEV90\tMAX (95%%)"
+ "\tMIN\tMEAN\tSTDDEV\tMAX\r\n");
+ }
+ printf("\tSP_Join:\t%d\t%d\t%.6f\t%.6f\t%.6f\t%.6f\t%.6f\t%.6f\t%.6f\t%.6f\t%.6f\t%.6f\t%.6f\r\n",
+ num_members, num_joins_leaves, sp_join_stats.quart1, sp_join_stats.median,
+ sp_join_stats.quart3, sp_join_stats.min5, sp_join_stats.mean90,
+ sp_join_stats.stddev90, sp_join_stats.max95, sp_join_stats.min,
+ sp_join_stats.mean, sp_join_stats.stddev, sp_join_stats.max);
+
+ printf("\tSP_Leave:\t%d\t%d\t%.6f\t%.6f\t%.6f\t%.6f\t%.6f\t%.6f\t%.6f\t%.6f\t%.6f\t%.6f\t%.6f\r\n",
+ num_members, num_joins_leaves, sp_leave_stats.quart1, sp_leave_stats.median,
+ sp_leave_stats.quart3, sp_leave_stats.min5, sp_leave_stats.mean90,
+ sp_leave_stats.stddev90, sp_leave_stats.max95, sp_leave_stats.min,
+ sp_leave_stats.mean, sp_leave_stats.stddev, sp_leave_stats.max);
+
+ printf("\tSP_Join + SP_Leave\t%d\t%d\t%.6f\t%.6f\t%.6f\t%.6f\t%.6f\t%.6f\t%.6f\t%.6f\t%.6f\t%.6f\t%.6f\r\n",
+ num_members, num_joins_leaves, sp_cmbo_stats.quart1, sp_cmbo_stats.median,
+ sp_cmbo_stats.quart3, sp_cmbo_stats.min5, sp_cmbo_stats.mean90,
+ sp_cmbo_stats.stddev90, sp_cmbo_stats.max95, sp_cmbo_stats.min,
+ sp_cmbo_stats.mean, sp_cmbo_stats.stddev, sp_cmbo_stats.max);
+
+ } else if (user_type == LOAD_MEMBER) {
+ if ((err = SP_join(mbox, group_name)) < 0) {
+ fprintf(stderr, "SP_join failure: ");
+ SP_error(err);
+ exit(1);
+ }
+ while (1) {
+ if ((mess_len = SP_receive(mbox, &service_type, sender,
+ MY_MAX_NUM_GROUPS, &num_groups, groups,
+ &mess_type, &endian_mismatch,
+ MY_MAX_MESS_SIZE, mess)) < 0) {
+ fprintf(stderr, "SP_receive failure: ");
+ SP_error(mess_len);
+ exit(1);
+ }
+ if (Is_reg_memb_mess(service_type)) {
+ if (Is_caused_leave_mess(service_type)) {
+ if (about_to_die) {
+ break;
+ }
+ } else if (!Is_caused_join_mess(service_type)) {
+ exit(fprintf(stderr, "Unexpected membership type: %d\n", service_type));
+ }
+ } else if (Is_safe_mess(service_type) && mess_type == DIE_MESS) {
+ about_to_die = 1;
+ }
+ }
+ printf("Success!\r\n");
+
+ } else {
+ fprintf(stderr, "Unknown user type: %d\n", user_type);
+ exit(printUsage(stderr));
+ }
+
+ if ((err = SP_disconnect(mbox)) < 0) {
+ fprintf(stderr, "SP_disconnect failure: ");
+ SP_error(err);
+ exit(1);
+ }
+ return 0;
+}
+
+
Added: trunk/flush/stats.c
===================================================================
--- trunk/flush/stats.c 2005-07-28 14:50:56 UTC (rev 252)
+++ trunk/flush/stats.c 2005-07-31 10:51:40 UTC (rev 253)
@@ -0,0 +1,81 @@
+#include <stdlib.h>
+#include <math.h>
+#include <sys/time.h>
+
+#include "stats.h"
+
+static int dbl_cmp(const void *arg1, const void *arg2)
+{
+ double a1 = *(double*) arg1, a2 = *(double*) arg2;
+
+ if (a1 < a2) {
+ return -1;
+
+ } else if (a1 > a2) {
+ return 1;
+ }
+ return 0;
+}
+
+int comp_stats(stats_results *ans, double *samples, size_t num_samples)
+{
+ size_t ind5 = num_samples / 20, ind95 = 19 * num_samples / 20, num_samples90 = ind95 - ind5 + 1;
+ size_t i;
+
+ /* total90, mean90, stddev90, min5, max95 are all computed from samples [5%, 95%] range */
+
+ if (num_samples == 0) {
+ return -1;
+ }
+ qsort(samples, num_samples, sizeof(double), dbl_cmp);
+
+ ans->min = samples[0];
+ ans->min5 = samples[ind5];
+ ans->max95 = samples[ind95];
+ ans->max = samples[num_samples - 1];
+
+ ans->quart1 = samples[num_samples / 4];
+ ans->median = samples[num_samples / 2];
+ ans->quart3 = samples[3 * num_samples / 4];
+
+ for (i = 0, ans->total = 0, ans->total90 = 0; i < ind5; ++i) {
+ ans->total += samples[i];
+ }
+ for (i = ind5; i <= ind95; ++i) {
+ ans->total90 += samples[i];
+ }
+ for (i = ind95 + 1; i < num_samples; ++i) {
+ ans->total += samples[i];
+ }
+ ans->mean90 = ans->total90 / num_samples90;
+ ans->mean = (ans->total + ans->total90) / num_samples;
+
+ if (num_samples != 1) {
+ for (i = 0, ans->stddev = 0, ans->stddev90 = 0; i < ind5; ++i) {
+ ans->stddev += (ans->mean - samples[i]) * (ans->mean - samples[i]);
+ }
+ for (i = ind5; i <= ind95; ++i) {
+ ans->stddev90 += (ans->mean90 - samples[i]) * (ans->mean90 - samples[i]);
+ ans->stddev += (ans->mean - samples[i]) * (ans->mean - samples[i]);
+ }
+ for (i = ind95 + 1; i < num_samples; ++i) {
+ ans->stddev += (ans->mean - samples[i]) * (ans->mean - samples[i]);
+ }
+ ans->stddev90 = sqrt(ans->stddev90 / (num_samples90 - 1));
+ ans->stddev = sqrt(ans->stddev / (num_samples - 1));
+
+ } else {
+ ans->stddev90 = 0;
+ ans->stddev = 0;
+ }
+ return 0;
+}
+
+double get_time_timeofday(void)
+{
+ struct timeval used;
+
+ gettimeofday(&used, 0);
+
+ return used.tv_sec * 1000.0 + used.tv_usec / 1000.0;
+}
Added: trunk/flush/stats.h
===================================================================
--- trunk/flush/stats.h 2005-07-28 14:50:56 UTC (rev 252)
+++ trunk/flush/stats.h 2005-07-31 10:51:40 UTC (rev 253)
@@ -0,0 +1,19 @@
+#ifndef stats_h_2001_07_06_14_19_22_jschultz_at_dfusion_net
+#define stats_h_2001_07_06_14_19_22_jschultz_at_dfusion_net
+
+#include <stddef.h>
+
+typedef struct {
+ double total, total90;
+ double mean, mean90;
+ double stddev, stddev90;
+ double min, min5, max, max95;
+ double quart1, median, quart3;
+
+} stats_results;
+
+int comp_stats(stats_results *ans, double *samples, size_t num_samples);
+
+double get_time_timeofday(void);
+
+#endif
Added: trunk/flush/user.c
===================================================================
--- trunk/flush/user.c 2005-07-28 14:50:56 UTC (rev 252)
+++ trunk/flush/user.c 2005-07-31 10:51:40 UTC (rev 253)
@@ -0,0 +1,423 @@
+/*
+ * The contents of this file are subject to the FLUSH SPREAD Non-Commercial
+ * License, Version 1.0 (the ``License''); you may not use this file except
+ * in compliance with the License. You may obtain a copy of the License at:
+ *
+ * http://www.cnds.jhu.edu/research/group/flush_spread/FLUSH_LICENSE
+ *
+ * or in the file ``FLUSH_LICENSE'' found in the root of this distribution.
+ *
+ * Software distributed under the License is distributed on an AS IS basis,
+ * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+ * for the specific language governing rights and limitations under the
+ * License.
+ *
+ * The Original Software is:
+ * The Flush Spread Library
+ *
+ * The Initial Developers of the Original Software are:
+ * Yair Amir, John Schultz and Jonathan Stanton
+ *
+ * All Rights Reserved.
+ *
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <fl.h>
+
+#ifdef USE_DMALLOC
+#include <dmalloc.h>
+#endif
+
+static char User[80];
+static char Spread_name[80];
+static char Private_group[MAX_GROUP_NAME];
+static mailbox Mbox;
+static int Num_sent;
+static int Previous_len;
+static int To_exit = 0;
+
+static void Print_menu();
+static void User_command();
+static void Read_message();
+static void Usage( int argc, char *argv[] );
+static void Bye();
+
+int main(int argc, char *argv[]) {
+ int ret, major, minor, patch;
+
+ Usage(argc, argv);
+
+ FL_version(&major, &minor, &patch);
+ printf("Flush library version is %d.%d.%d\n", major, minor, patch);
+
+ ret = FL_connect( Spread_name, User, LOW_PRIORITY, &Mbox, Private_group );
+ if( ret < 0 ) {
+ FL_error( ret );
+ Bye();
+ }
+ printf("User: connected to %s with private group %s\n", Spread_name, Private_group );
+
+ Print_menu();
+
+ printf("\nUser> ");
+ fflush(stdout);
+
+ Num_sent = 0;
+
+ for(;;)
+ User_command();
+
+ return 0;
+}
+
+static void User_command()
+{
+ char command[130];
+ char mess[102400];
+ char group[80];
+ char groups[10][MAX_GROUP_NAME];
+ int num_groups;
+ int mess_len;
+ int ret;
+ int i;
+
+ for (i = 0; i < sizeof(command); i++)
+ command[i] = 0;
+
+ if (fgets(command, 130, stdin) == 0)
+ Bye();
+
+ switch(command[0]) {
+ case 'j':
+ ret = sscanf( &command[2], "%s", group );
+ if( ret < 1 ) {
+ printf(" invalid group \n");
+ break;
+ }
+ ret = FL_join( Mbox, group );
+ if( ret < 0 ) FL_error( ret );
+ break;
+
+ case 'l':
+ ret = sscanf( &command[2], "%s", group );
+ if( ret < 1 )
+ {
+ printf(" invalid group \n");
+ break;
+ }
+ ret = FL_leave( Mbox, group );
+ if( ret < 0 ) FL_error( ret );
+
+ break;
+
+ case 'g': {
+ int i;
+
+ if ((num_groups = sscanf(&command[2], "%s", groups[0])) < 1) {
+ printf("invalid group\n");
+ break;
+ }
+ printf("enter number of receivers: ");
+ fflush(stdout);
+ if (fgets(mess, 200, stdin) == 0 ||
+ sscanf(mess, "%d", &num_groups) != 1 ||
+ num_groups < 1 || num_groups > 9) {
+ printf("invalid number of receivers, must be an integer in the range [1, 9]\n");
+ break;
+ }
+ for (i = 1; i <= num_groups; ++i) {
+ printf("enter recvr %d: ", i);
+ fflush(stdout);
+ if (fgets(mess, MAX_GROUP_NAME, stdin) == 0 || sscanf(mess, "%s", groups[i]) != 1)
+ Bye();
+ }
+ printf("enter message: ");
+ fflush(stdout);
+ if (fgets(mess, 200, stdin) == 0) Bye();
+ mess_len = strlen(mess);
+
+ printf("user.c: subgroupcasting message of size %d to group %s with receivers:\n", mess_len, groups[0]);
+ for (i = 1; i <= num_groups; ++i)
+ printf("%s\n", groups[i]);
+
+ if ((ret = FL_subgroupcast(Mbox, SAFE_MESS, groups[0], num_groups, groups+1, 0, mess_len, mess)) != mess_len) {
+ if (ret < 0) {
+ FL_error(ret);
+ Bye();
+ } else
+ printf("Wierd return from FL_subgroupcast %d, should be %d\n", ret, mess_len);
+ }
+ Num_sent++;
+ break;
+ }
+
+ case 's':
+ num_groups = sscanf(&command[2], "%s", groups[0]);
+ if( num_groups < 1 )
+ {
+ printf(" invalid group \n");
+ break;
+ }
+ printf("enter message: ");
+ ret = (int) fgets( mess, 200, stdin );
+ if( ret==0 ) Bye();
+ mess_len = strlen( mess );
+
+ printf("user.c : multicasting message of size %d:\n'%s'\n", mess_len, mess);
+
+ ret = FL_multicast( Mbox, SAFE_MESS, groups[0], 1, mess_len, mess );
+ if( ret < 0 )
+ {
+ FL_error( ret );
+ Bye();
+ }
+ Num_sent++;
+ break;
+
+ case 'b':
+ ret = sscanf( &command[2], "%s", group );
+ if( ret != 1 ) strcpy( group, "dummy_group_name" );
+ printf("enter size of each message: ");
+ ret = (int) fgets( mess, 200, stdin );
+ if( ret==0 ) Bye();
+ ret = sscanf(mess, "%d", &mess_len );
+ if( ret !=1 ) mess_len = Previous_len;
+ if( mess_len < 0 ) mess_len = 0;
+ Previous_len = mess_len;
+ printf("sending 10 messages of %d bytes\n", mess_len );
+ for( i=0; i<10; i++ )
+ {
+ Num_sent++;
+ sprintf( mess, "mess num %d", Num_sent );
+ ret= FL_multicast( Mbox, FIFO_MESS, group, 2, mess_len, mess );
+
+ if( ret < 0 )
+ {
+ FL_error( ret );
+ Bye();
+ }
+ printf("sent message %d (total %d)\n", i+1, Num_sent );
+ }
+ break;
+ case 'r':
+
+ Read_message();
+ break;
+
+ case 'p':
+
+ ret = FL_poll( Mbox );
+ printf("Polling says: %d\n", ret );
+ break;
+
+ /*
+ case 'e':
+
+ E_attach_fd( Mbox, Read_message, 0, HIGH_PRIORITY );
+ break;
+
+ case 'd':
+
+ E_detach_fd( Mbox );
+ break;
+
+ */
+ case 'f':
+ ret = sscanf( &command[2], "%s", group);
+ if( ret < 1 )
+ {
+ printf(" invalid group \n");
+ break;
+ }
+ ret = FL_flush(Mbox, group);
+
+ printf("Sent a FLUSH_OK message to group '%s'\n", group);
+
+ if (ret < 0) {
+ FL_error(ret);
+ Bye();
+ }
+ break;
+
+ case 'q':
+ Bye();
+ break;
+
+ default:
+ printf("\nUnknown commnad\n");
+ Print_menu();
+
+ break;
+ }
+ printf("\nUser> ");
+ fflush(stdout);
+}
+
+static void Print_menu()
+{
+ printf("\n");
+ printf("==========\n");
+ printf("User Menu:\n");
+ printf("----------\n");
+ printf("\n");
+ printf("\tj <group> -- join a group\n");
+ printf("\tl <group> -- leave a group\n");
+ printf("\n");
+ printf("\ts <group> -- send a message\n");
+ printf("\tg <group> -- send a subgroup message\n");
+ printf("\tb <group> -- send a burst of messages\n");
+ printf("\n");
+ printf("\tr -- receive a message (stuck) \n");
+ printf("\tp -- poll for a message \n");
+ /*
+ printf("\te -- enable asynchonous read (default)\n");
+ printf("\td -- disable asynchronous read \n");
+ */
+ printf("\tf <group> -- send a FLUSH_OK message\n");
+ printf("\n");
+ printf("\tq -- quit\n");
+ fflush(stdout);
+}
+
+static void Read_message()
+{
+
+ static char mess[102400];
+ char sender[MAX_GROUP_NAME];
+ char target_groups[100][MAX_GROUP_NAME];
+ group_id *grp_id;
+ int32 *num_vs;
+ char *vs_members;
+ int num_groups;
+ int num_bytes;
+ int service_type;
+ int16 mess_type;
+ int endian_mismatch;
+ int i;
+ int ret;
+ int more_messes;
+
+ ret = FL_receive( Mbox, &service_type, sender, 100, &num_groups, target_groups,
+ &mess_type, &endian_mismatch, sizeof(mess), mess, &more_messes);
+ printf("\n============================\n");
+ if( ret < 0 )
+ {
+ if( ! To_exit )
+ {
+ FL_error( ret );
+ printf("\n============================\n");
+ printf("\nBye.\n");
+ }
+ exit( 0 );
+ }
+ if( Is_regular_mess( service_type ) )
+ {
+ mess[ret] = 0;
+ if ( Is_unreliable_mess( service_type ) ) printf("received UNRELIABLE ");
+ else if( Is_reliable_mess( service_type ) ) printf("received RELIABLE ");
+ else if( Is_fifo_mess( service_type ) ) printf("received FIFO ");
+ else if( Is_causal_mess( service_type ) ) printf("received CAUSAL ");
+ else if( Is_agreed_mess( service_type ) ) printf("received AGREED ");
+ else if( Is_safe_mess( service_type ) ) printf("received SAFE ");
+
+ printf("message from %s, of type %d, (endian %d) to %d groups \n(%d bytes): %s\n",
+ sender, mess_type, endian_mismatch, num_groups, ret, mess );
+ printf("Message is:\n'%s'\n", mess);
+ if (Is_subgroup_mess(service_type)) {
+ int i;
+
+ printf("Received a subgroupcast message to group %s: receivers:\n", target_groups[num_groups - 1]);
+ for (i = 0; i < num_groups - 1; ++i)
+ printf("\t%s\n", target_groups[i]);
+ }
+ } else if( Is_membership_mess( service_type ) ){
+ if ( Is_reg_memb_mess( service_type ) )
+ {
+ num_bytes = 0;
+ grp_id = (group_id *)&mess[num_bytes];
+ num_bytes += sizeof( group_id );
+ num_vs = (int32 *)&mess[num_bytes];
+ num_bytes += sizeof( int32 );
+ vs_members = &mess[num_bytes];
+
+ printf("Received REGULAR membership for group %s with %d members, where I am member %d:\n",
+ sender, num_groups, mess_type );
+
+ for( i=0; i < num_groups; i++ )
+ printf("\t%s\n", &target_groups[i][0]);
+
+ printf("grp id is %d %d %d\n", grp_id->id[0], grp_id->id[1], grp_id->id[2]);
+
+ if( Is_caused_join_mess( service_type ) )
+ {
+ printf("Due to the JOIN of %s\n", vs_members );
+ }else if( Is_caused_leave_mess( service_type ) ){
+ printf("Due to the LEAVE of %s\n", vs_members);
+ }else if( Is_caused_disconnect_mess( service_type ) ){
+ printf("Due to the DISCONNECT of %s\n", vs_members );
+ }else if( Is_caused_network_mess( service_type ) ){
+ printf("Due to NETWORK change. ");
+ printf("VS set has %d members:\n", *num_vs );
+ for( i=0; i < *num_vs; i++, vs_members+= MAX_GROUP_NAME )
+ printf("\t%s\n", vs_members );
+ }
+ }else if( Is_transition_mess( service_type ) ) {
+ printf("received TRANSITIONAL membership for group %s\n", sender );
+ }else if( Is_caused_leave_mess( service_type ) ){
+ printf("received membership message that left group %s\n", sender );
+ }else printf("received incorrecty membership message of type %d\n", service_type );
+ }else if (Is_flush_req_mess(service_type)) {
+ printf("received a FLUSH_REQ message for group %s\n", sender);
+ }else printf("received message of unknown message type %d with ret %d\n", service_type, ret);
+
+ printf("There are %d buffered messages waiting to be delivered\n", more_messes);
+
+ printf("\n");
+ printf("User> ");
+ fflush(stdout);
+}
+
+static void Usage(int argc, char *argv[])
+{
+
+ sprintf( User, "user" );
+ sprintf( Spread_name, "3333 at localhost");
+ while( --argc > 0 )
+ {
+ argv++;
+
+ if( !strncmp( *argv, "-u", 2 ) )
+ {
+ strcpy( User, argv[1] );
+ argc--; argv++;
+ }else if( !strncmp( *argv, "-s", 2 ) ){
+ strcpy( Spread_name, argv[1] );
+ argc--; argv++;
+ }else{
+ printf( "Usage: user\n%s\n%s\n",
+ "\t[-u <user name>] : unique (in this machine) user name",
+ "\t[-s <address>] : either port or port at machine");
+ exit( 0 );
+ }
+ }
+}
+
+static void Bye()
+{
+ To_exit = 1;
+
+ printf("\nBye.\n");
+
+ printf("Calling FL_disconnect(mbox = %d)\n", Mbox);
+
+ FL_disconnect( Mbox );
+ exit( 0 );
+}
+
+
+
+
+
Added: trunk/include/fl.h
===================================================================
--- trunk/include/fl.h 2005-07-28 14:50:56 UTC (rev 252)
+++ trunk/include/fl.h 2005-07-31 10:51:40 UTC (rev 253)
@@ -0,0 +1,114 @@
+/*
+ * The contents of this file are subject to the FLUSH SPREAD Non-Commercial
+ * License, Version 1.0 (the ``License''); you may not use this file except
+ * in compliance with the License. You may obtain a copy of the License at:
+ *
+ * http://www.cnds.jhu.edu/research/group/flush_spread/FLUSH_LICENSE
+ *
+ * or in the file ``FLUSH_LICENSE'' found in the root of this distribution.
+ *
+ * Software distributed under the License is distributed on an AS IS basis,
+ * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+ * for the specific language governing rights and limitations under the
+ * License.
+ *
+ * The Original Software is:
+ * The Flush Spread Library
+ *
+ * The Initial Developers of the Original Software are:
+ * Yair Amir, John Schultz and Jonathan Stanton
+ *
+ * All Rights Reserved.
+ *
+ */
+
+#ifndef fl_h_2000_03_20_14_36_26_jschultz_at_cnds_jhu_edu
+#define fl_h_2000_03_20_14_36_26_jschultz_at_cnds_jhu_edu
+
+#include <sp.h>
+
+/* FL service types */
+#define DONT_BLOCK 0x10000000
+#define FLUSH_REQ_MESS 0x20000000
+#define SUBGROUP_CAST 0x40000000
+
+/* FL service query macros */
+#define Is_flush_req_mess(serv) (((serv) & FLUSH_REQ_MESS) != 0)
+#define Is_subgroup_mess(serv) (((serv) & SUBGROUP_CAST) != 0)
+
+/* FL error codes */
+#define ILLEGAL_PARAM -24
+#define WOULD_BLOCK -25
+#define ILLEGAL_MESSAGE_TYPE -26
+#define ILLEGAL_STATE -27
+#define ILLEGAL_RECEIVERS -28
+
+/* maximum # of usable scatter elements for FL msgs */
+#define FL_MAX_SCATTER_ELEMENTS (MAX_CLIENT_SCATTER_ELEMENTS - 1)
+
+/* minimum message type a user is allowed to use - less than this is illegal */
+#define FL_MIN_LEGAL_MESS_TYPE ((int16) -32765)
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+int FL_lib_init(void);
+
+void FL_version(int *major_ver, int *minor_ver, int *patch_ver);
+
+int FL_connect(const char *daemon_name, const char *user_name, int priority,
+ mailbox *mbox, char *private_name);
+
+int FL_disconnect(mailbox mbox);
+
+int FL_join(mailbox mbox, const char *group_name);
+
+int FL_leave(mailbox mbox, const char *group_name);
+
+int FL_flush(mailbox mbox, const char *group_name);
+
+int FL_unicast(mailbox mbox, service serv_type, const char *group_name,
+ const char *recvr_name, int16 mess_type, int mess_len, const char *mess);
+
+int FL_scat_unicast(mailbox mbox, service serv_type, const char *group_name,
+ const char *recvr_name, int16 mess_type, const scatter *scat);
+
+int FL_subgroupcast(mailbox mbox, service serv_type, const char *group_name,
+ int num_recvrs, char recvr_names[][MAX_GROUP_NAME],
+ int16 mess_type, int mess_len, const char *mess);
+
+int FL_scat_subgroupcast(mailbox mbox, service serv_type, const char *group_name,
+ int num_recvrs, char recvr_names[][MAX_GROUP_NAME],
+ int16 mess_type, const scatter *scat);
+
+int FL_multicast(mailbox mbox, service serv_type, const char *group_name,
+ int16 mess_type, int mess_len, const char *mess);
+
+int FL_scat_multicast(mailbox mbox, service serv_type, const char *group_name,
+ int16 mess_type, const scatter *scat_mess);
+
+int FL_receive(mailbox mbox, service *serv_type, char *sender_name,
+ int max_groups, int *num_groups, char group_names[][MAX_GROUP_NAME],
+ int16 *mess_type, int *endian_mismatch, int max_mess_len,
+ char *mess, int *more_msgs);
+
+int FL_scat_receive(mailbox mbox, service *serv_type, char *sender_name,
+ int max_groups, int *num_groups, char group_names[][MAX_GROUP_NAME],
+ int16 *mess_type, int *endian_mismatch, scatter *scat_mess,
+ int *more_msgs);
+
+int FL_more_msgs(mailbox mbox);
+
+int FL_poll(mailbox mbox);
+
+void FL_error(int error_code);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
+
+
+
More information about the Spread-cvs
mailing list