Index: test/testcmanquorum1.c =================================================================== --- test/testcmanquorum1.c (revision 0) +++ test/testcmanquorum1.c (revision 0) @@ -0,0 +1,124 @@ +#include +#include +#include +#include +#include +#include +#include + +static quorum_handle_t handle; + +static char *node_state(int state) +{ + switch (state) { + case NODESTATE_JOINING: + return "Joining"; + break; + case NODESTATE_MEMBER: + return "Member"; + break; + case NODESTATE_DEAD: + return "Dead"; + break; + case NODESTATE_LEAVING: + return "Leaving"; + break; + case NODESTATE_DISALLOWED: + return "Disallowed"; + break; + default: + return "UNKNOWN"; + break; + } +} + +static void quorum_notification_fn( + quorum_handle_t handle, + uint32_t quorate, + uint32_t node_list_entries, + quorum_node_t node_list[] + ) +{ + int i; + + printf("quorum notification called \n"); + printf(" quorate = %d\n", quorate); + printf(" number of nodes = %d\n", node_list_entries); + + for (i = 0; i< node_list_entries; i++) { + printf(" %d: %s\n", node_list[i].nodeid, node_state(node_list[i].state)); + } + printf("\n"); +} + + +int main(int argc, char *argv[]) +{ + struct quorum_info info; + quorum_callbacks_t callbacks; + int err; + + if (argc > 1 && strcmp(argv[1], "-h")==0) { + fprintf(stderr, "usage: %s [new-expected] [new-votes]\n"); + return 0; + } + + callbacks.quorum_notify_fn = quorum_notification_fn; + if ( (err=quorum_initialize(&handle, &callbacks)) != QUORUM_OK) + fprintf(stderr, "quorum_initialize FAILED: %d\n", err); + + if (quorum_trackstart(handle, SA_TRACK_CHANGES) != QUORUM_OK) + fprintf(stderr, "quorum_trackstart FAILED: %d\n", err); + + if ( (err=quorum_getinfo(handle, 0, &info)) != QUORUM_OK) + fprintf(stderr, "quorum_getinfo FAILED: %d\n", err); + else { + printf("node votes %d\n", info.node_votes); + printf("expected votes %d\n", info.node_expected_votes); + printf("highest expected %d\n", info.highest_expected); + printf("total votes %d\n", info.total_votes); + printf("quorum %d\n", info.quorum); + printf("flags "); + if (info.flags & QUORUM_INFO_FLAG_DIRTY) printf("Dirty "); + if (info.flags & QUORUM_INFO_FLAG_DISALLOWED) printf("Disallowed "); + if (info.flags & QUORUM_INFO_FLAG_TWONODE) printf("2Node "); + if (info.flags & QUORUM_INFO_FLAG_QUORATE) printf("Quorate "); + printf("\n"); + } + + if (argc >= 2 && atoi(argv[1])) { + if ( (err=quorum_setexpected(handle, atoi(argv[1]))) != QUORUM_OK) + fprintf(stderr, "set expected votes FAILED: %d\n", err); + } + if (argc >= 3 && atoi(argv[2])) { + if ( (err=quorum_setvotes(handle, 0, atoi(argv[2]))) != QUORUM_OK) + fprintf(stderr, "set votes FAILED: %d\n", err); + } + + if (argc >= 2) { + if ( (err=quorum_getinfo(handle, 0, &info)) != QUORUM_OK) + fprintf(stderr, "quorum_getinfo2 FAILED: %d\n", err); + else { + printf("-------------------\n"); + printf("node votes %d\n", info.node_votes); + printf("expected votes %d\n", info.node_expected_votes); + printf("highest expected %d\n", info.highest_expected); + printf("total votes %d\n", info.total_votes); + printf("quorum %d\n", info.quorum); + printf("flags "); + if (info.flags & QUORUM_INFO_FLAG_DIRTY) printf("Dirty "); + if (info.flags & QUORUM_INFO_FLAG_DISALLOWED) printf("Disallowed "); + if (info.flags & QUORUM_INFO_FLAG_TWONODE) printf("2Node "); + if (info.flags & QUORUM_INFO_FLAG_QUORATE) printf("Quorate "); + printf("\n"); + } + } + + printf("Waiting for quorum events, press ^C to finish\n"); + printf("-------------------\n"); + + while (1) + quorum_dispatch(handle, QUORUM_DISPATCH_ALL); + + return 0; +} Index: test/testcmanquorum2.c =================================================================== --- test/testcmanquorum2.c (revision 0) +++ test/testcmanquorum2.c (revision 0) @@ -0,0 +1,63 @@ +#include +#include +#include +#include +#include +#include +#include + +static quorum_handle_t handle; + + +static void print_info(int ok_to_fail) +{ + struct quorum_qdisk_info qinfo; + int err; + + if ( (err=quorum_qdisk_getinfo(handle, &qinfo)) != QUORUM_OK) + fprintf(stderr, "quorum_qdisk_getinfo error %d: %s\n", err, ok_to_fail?"OK":"FAILED"); + else { + printf("qdisk votes %d\n", qinfo.votes); + printf("state %d\n", qinfo.state); + printf("name %s\n", qinfo.name); + printf("\n"); + } +} + +int main(int argc, char *argv[]) +{ + int pollcount=0, polltime=1; + int err; + + if ( (err=quorum_initialize(&handle, NULL)) != QUORUM_OK) { + fprintf(stderr, "quorum_initialize FAILED: %d\n", err); + return -1; + } + + print_info(1); + + if (argc >= 2 && atoi(argv[1])) { + pollcount = atoi(argv[1]); + } + if (argc >= 3 && atoi(argv[2])) { + polltime = atoi(argv[2]); + } + + if (argc >= 2) { + if ( (err=quorum_qdisk_register(handle, "QDISK", 4)) != QUORUM_OK) + fprintf(stderr, "qdisk_register FAILED: %d\n", err); + + while (pollcount--) { + print_info(0); + if ((err=quorum_qdisk_poll(handle, 1)) != QUORUM_OK) + fprintf(stderr, "qdisk poll FAILED: %d\n", err); + print_info(0); + sleep(polltime); + } + if ((err= quorum_qdisk_unregister(handle)) != QUORUM_OK) + fprintf(stderr, "qdisk unregister FAILED: %d\n", err); + } + print_info(1); + + return 0; +} Index: test/Makefile =================================================================== --- test/Makefile (revision 1663) +++ test/Makefile (working copy) @@ -37,9 +37,9 @@ override LDFLAGS += -lnsl -lsocket -lrt endif -LIBRARIES= ../lib/libevs.a ../lib/libcpg.a ../lib/libcfg.a ../lib/libconfdb.a +LIBRARIES= ../lib/libevs.a ../lib/libcpg.a ../lib/libcfg.a ../lib/libconfdb.a ../lib/libquorum.a LIBS = $(LIBRARIES) -BINARIES= testevs evsbench testcpg testcpg2 cpgbench testconfdb +BINARIES= testevs evsbench testcpg testcpg2 cpgbench testconfdb testquorum1 testquorum2 #override CFLAGS += -I../include override LDFLAGS += -L../lib @@ -73,6 +73,11 @@ testconfdb: testconfdb.o $(LIBRARIES) $(CC) $(LDFLAGS) -o testconfdb testconfdb.o $(LIBS) -rdynamic +testquorum1: testquorum1.o $(LIBRARIES) + $(CC) $(LDFLAGS) -o testquorum1 testquorum1.o $(LIBS) +testquorum2: testquorum2.o $(LIBRARIES) + $(CC) $(LDFLAGS) -o testquorum2 testquorum2.o $(LIBS) + logsys_s: logsys_s.o logsys_s1.o logsys_s2.o ../exec/liblogsys.a $(CC) -o logsys_s logsys_s.o logsys_s1.o logsys_s2.o ../exec/liblogsys.a $(LDFLAGS) Index: include/corosync/ipc_cmanquorum.h =================================================================== --- include/corosync/ipc_cmanquorum.h (revision 0) +++ include/corosync/ipc_cmanquorum.h (revision 0) @@ -0,0 +1,155 @@ +/* + * Copyright (c) 2008 Red Hat, Inc. + * + * All rights reserved. + * + * Author: Christine Caulfield (ccaulfie@redhat.com) + * + * This software licensed under BSD license, the text of which follows: + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * - Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * - Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * - Neither the name of the MontaVista Software, Inc. nor the names of its + * contributors may be used to endorse or promote products derived from this + * software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ +#ifndef IPC_QUORUM_H_DEFINED +#define IPC_QUORUM_H_DEFINED + +#include +#include "saAis.h" +#include "corosync/ipc_gen.h" + +// ILLEGAL value!! +#define QUORUM_SERVICE 15 + +#define QUORUM_MAX_QDISK_NAME_LEN 255 + + +enum req_quorum_types { + MESSAGE_REQ_QUORUM_GETINFO = 0, + MESSAGE_REQ_QUORUM_SETEXPECTED, + MESSAGE_REQ_QUORUM_SETVOTES, + MESSAGE_REQ_QUORUM_QDISK_REGISTER, + MESSAGE_REQ_QUORUM_QDISK_UNREGISTER, + MESSAGE_REQ_QUORUM_QDISK_POLL, + MESSAGE_REQ_QUORUM_QDISK_GETINFO, + MESSAGE_REQ_QUORUM_SETDIRTY, + MESSAGE_REQ_QUORUM_KILLNODE, + MESSAGE_REQ_QUORUM_LEAVING, + MESSAGE_REQ_QUORUM_TRACKSTART, + MESSAGE_REQ_QUORUM_TRACKSTOP +}; + +enum res_quorum_types { + MESSAGE_RES_QUORUM_STATUS = 0, + MESSAGE_RES_QUORUM_GETINFO, + MESSAGE_RES_QUORUM_QDISK_GETINFO, + MESSAGE_RES_QUORUM_TRACKSTART, + MESSAGE_RES_QUORUM_NOTIFICATION +}; + +struct req_lib_quorum_setvotes { + mar_req_header_t header __attribute__((aligned(8))); + unsigned int votes; + int nodeid; +}; + +struct req_lib_quorum_qdisk_register { + mar_req_header_t header __attribute__((aligned(8))); + int votes; + char name[QUORUM_MAX_QDISK_NAME_LEN]; +}; + +struct req_lib_quorum_qdisk_poll { + mar_req_header_t header __attribute__((aligned(8))); + int state; +}; + +struct req_lib_quorum_setexpected { + mar_req_header_t header __attribute__((aligned(8))); + int expected_votes; +}; + +struct req_lib_quorum_trackstart { + mar_req_header_t header __attribute__((aligned(8))); + unsigned int track_flags; +}; + +struct req_lib_quorum_general { + mar_req_header_t header __attribute__((aligned(8))); +}; + +#define QUORUM_REASON_KILL_REJECTED 1 +#define QUORUM_REASON_KILL_APPLICATION 2 +#define QUORUM_REASON_KILL_REJOIN 3 + +struct req_lib_quorum_killnode { + mar_req_header_t header __attribute__((aligned(8))); + int nodeid; + unsigned int reason; +}; + +struct req_lib_quorum_getinfo { + mar_req_header_t header __attribute__((aligned(8))); + int nodeid; +}; + +struct res_lib_quorum_status { + mar_res_header_t header __attribute__((aligned(8))); +}; + +#define QUORUM_INFO_FLAG_DIRTY 1 +#define QUORUM_INFO_FLAG_DISALLOWED 2 +#define QUORUM_INFO_FLAG_TWONODE 4 +#define QUORUM_INFO_FLAG_QUORATE 8 + +struct res_lib_quorum_getinfo { + mar_res_header_t header __attribute__((aligned(8))); + int nodeid; + unsigned int votes; + unsigned int expected_votes; + unsigned int highest_expected; + unsigned int total_votes; + unsigned int quorum; + unsigned int flags; +}; + +struct res_lib_quorum_qdisk_getinfo { + mar_res_header_t header __attribute__((aligned(8))); + int votes; + int state; + char name[QUORUM_MAX_QDISK_NAME_LEN]; +}; + +struct quorum_node { + mar_uint32_t nodeid; + mar_uint32_t state; +}; + +struct res_lib_quorum_notification { + mar_res_header_t header __attribute__((aligned(8))); + mar_uint32_t quorate __attribute__((aligned(8))); + mar_uint32_t node_list_entries __attribute__((aligned(8))); + struct quorum_node node_list[] __attribute__((aligned(8))); +}; + +#endif Index: include/corosync/cmanquorum.h =================================================================== --- include/corosync/cmanquorum.h (revision 0) +++ include/corosync/cmanquorum.h (revision 0) @@ -0,0 +1,224 @@ +/* + * Copyright (c) 2008 Red Hat, Inc. + * + * All rights reserved. + * + * Author: Christine Caulfield (ccaulfi@redhat.com) + * + * This software licensed under BSD license, the text of which follows: + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * - Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * - Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * - Neither the name of the MontaVista Software, Inc. nor the names of its + * contributors may be used to endorse or promote products derived from this + * software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ +#ifndef COROSYNC_QUORUM_H_DEFINED +#define COROSYNC_QUORUM_H_DEFINED + +typedef uint64_t quorum_handle_t; + +typedef enum { + QUORUM_OK = 1, + QUORUM_ERR_LIBRARY = 2, + QUORUM_ERR_TIMEOUT = 5, + QUORUM_ERR_TRY_AGAIN = 6, + QUORUM_ERR_INVALID_PARAM = 7, + QUORUM_ERR_NO_MEMORY = 8, + QUORUM_ERR_BAD_HANDLE = 9, + QUORUM_ERR_ACCESS = 11, + QUORUM_ERR_NOT_EXIST = 12, + QUORUM_ERR_EXIST = 14, + QUORUM_ERR_NOT_SUPPORTED = 20, + QUORUM_ERR_SECURITY = 29 +} quorum_error_t; + + +#define QUORUM_MAX_QDISK_NAME_LEN 255 + +#define QUORUM_INFO_FLAG_DIRTY 1 +#define QUORUM_INFO_FLAG_DISALLOWED 2 +#define QUORUM_INFO_FLAG_TWONODE 4 +#define QUORUM_INFO_FLAG_QUORATE 8 + +#define NODESTATE_JOINING 1 +#define NODESTATE_MEMBER 2 +#define NODESTATE_DEAD 3 +#define NODESTATE_LEAVING 4 +#define NODESTATE_DISALLOWED 5 + + +/** @} */ + +struct quorum_info { + int node_id; + unsigned int node_votes; + unsigned int node_expected_votes; + unsigned int highest_expected; + unsigned int total_votes; + unsigned int quorum; + unsigned int flags; +}; + +struct quorum_qdisk_info { + int votes; + int state; + char name[QUORUM_MAX_QDISK_NAME_LEN]; +}; + +typedef enum { + QUORUM_DISPATCH_ONE, + QUORUM_DISPATCH_ALL, + QUORUM_DISPATCH_BLOCKING +} quorum_dispatch_t; + +typedef struct { + uint32_t nodeid; + uint32_t state; +} quorum_node_t; + + +typedef void (*quorum_notification_fn_t) ( + quorum_handle_t handle, + uint32_t quorate, + uint32_t node_list_entries, + quorum_node_t node_list[] + ); + +typedef struct { + quorum_notification_fn_t quorum_notify_fn; +} quorum_callbacks_t; + + +/* + * Create a new quorum connection + */ +quorum_error_t quorum_initialize ( + quorum_handle_t *handle, + quorum_callbacks_t *callbacks); + +/* + * Close the quorum handle + */ +quorum_error_t quorum_finalize ( + quorum_handle_t handle); + + +/* + * Dispatch messages and configuration changes + */ +quorum_error_t quorum_dispatch ( + quorum_handle_t handle, + quorum_dispatch_t dispatch_types); + + +/* + * Get quorum information. + */ +quorum_error_t quorum_getinfo ( + quorum_handle_t handle, + int nodeid, + struct quorum_info *info); + +/* + * set expected_votes + */ +quorum_error_t quorum_setexpected ( + quorum_handle_t handle, + unsigned int expected_votes); + +/* + * set votes for a node + */ +quorum_error_t quorum_setvotes ( + quorum_handle_t handle, + int nodeid, + unsigned int votes); + +/* + * Register a quorum device + * it will be DEAD until polled + */ +quorum_error_t quorum_qdisk_register ( + quorum_handle_t handle, + char *name, + unsigned int votes); + +/* + * Unregister a quorum device + */ +quorum_error_t quorum_qdisk_unregister ( + quorum_handle_t handle); + +/* + * Poll a quorum device + */ +quorum_error_t quorum_qdisk_poll ( + quorum_handle_t handle, + int state); + +/* + * Get quorum device information + */ +quorum_error_t quorum_qdisk_getinfo ( + quorum_handle_t handle, + struct quorum_qdisk_info *info); + +/* + * Set the dirty bit for this node + */ +quorum_error_t quorum_setdirty ( + quorum_handle_t handle); + +/* + * Force a node to exit + */ +quorum_error_t quorum_killnode ( + quorum_handle_t handle, + int nodeid, + unsigned int reason); + +/* Track node and quorum changes */ +quorum_error_t quorum_trackstart ( + quorum_handle_t handle, + unsigned int flags ); + +quorum_error_t quorum_trackstop ( + quorum_handle_t handle); + +/* + * Set our LEAVING flag. we should exit soon after this + */ +quorum_error_t quorum_leaving ( + quorum_handle_t handle); + +/* + * Save and retrieve private data/context + */ +quorum_error_t quorum_context_get ( + quorum_handle_t handle, + void **context) + +quorum_error_t quorum_context_set ( + quorum_handle_t handle, + void *context) + +#endif /* COROSYNC_QUORUM_H_DEFINED */ Index: services/cmanquorum.c =================================================================== --- services/cmanquorum.c (revision 0) +++ services/cmanquorum.c (revision 0) @@ -0,0 +1,1398 @@ +/* + * Copyright (c) 2006-2008 Red Hat, Inc. + * + * All rights reserved. + * + * Author: Christine Caulfield (ccaulfie@redhat.com) + * + * This software licensed under BSD license, the text of which follows: + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * - Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * - Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * - Neither the name of the MontaVista Software, Inc. nor the names of its + * contributors may be used to endorse or promote products derived from this + * software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ +#ifndef COROSYNC_BSD +#include +#endif +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "quorum.h" + +#define QUORUM_MAJOR_VERSION 6 +#define QUORUM_MINOR_VERSION 3 +#define QUORUM_PATCH_VERSION 0 + + /* Silly default to prevent accidents! */ +#define DEFAULT_EXPECTED 1024 +#define DEFAULT_QDEV_POLL 10000 + +LOGSYS_DECLARE_SUBSYS ("QUORUM", LOG_INFO); + +enum quorum_message_req_types { + MESSAGE_REQ_EXEC_QUORUM_NODEINFO = 0, + MESSAGE_REQ_EXEC_QUORUM_RECONFIGURE = 1, + MESSAGE_REQ_EXEC_QUORUM_KILLNODE = 2, +}; + +#define NODE_FLAGS_QDISK 1 +#define NODE_FLAGS_REMOVED 2 +#define NODE_FLAGS_BEENDOWN 4 +#define NODE_FLAGS_SEESDISALLOWED 8 +#define NODE_FLAGS_US 16 +#define NODE_FLAGS_DIRTY 32 + +typedef enum { NODESTATE_JOINING=1, NODESTATE_MEMBER, + NODESTATE_DEAD, NODESTATE_LEAVING, NODESTATE_AISONLY } nodestate_t; + + +/* This structure is tacked onto the start of a cluster message packet for our + * own nefarious purposes. */ +struct q_protheader { + unsigned char tgtport; /* Target port number */ + unsigned char srcport; /* Source (originating) port number */ + unsigned short pad; + unsigned int flags; + int srcid; /* Node ID of the sender */ + int tgtid; /* Node ID of the target */ +}; + +struct cluster_node { + int flags; + int node_id; + unsigned int expected_votes; + unsigned int votes; + + nodestate_t state; + + struct timeval last_hello; /* Only used for quorum devices */ + + struct list_head list; +}; + +#define QUORUM_FLAG_FEATURE_DISALLOWED 1 + +static int quorum_flags; +static int quorum; +static int cluster_is_quorate; +static int first_trans = 1; +static unsigned int two_node; +static unsigned int quorumdev_poll = DEFAULT_QDEV_POLL; + +static struct cluster_node *us; +static struct cluster_node *quorum_device = NULL; +static char quorum_device_name[QUORUM_MAX_QDISK_NAME_LEN]; +static corosync_timer_handle_t quorum_device_timer; +static struct list_head cluster_members_list; +static struct corosync_api_v1 *corosync_api; +static struct list_head trackers_list; + +#define max(a,b) (((a) > (b)) ? (a) : (b)) +static struct cluster_node *find_node_by_nodeid(int nodeid); +static struct cluster_node *allocate_node(int nodeid); +static char *kill_reason(int reason); + +/* The name 'CMAN' is for backwards compatibility */ +static corosync_tpg_handle group_handle; +static struct corosync_tpg_group quorum_group[1] = { + { .group = "CMAN", .group_len = 4}, +}; + +#define list_iterate(v, head) \ + for (v = (head)->next; v != head; v = v->next) + +struct quorum_pd { + unsigned char track_flags; + int tracking_enabled; + struct list_head list; + void *conn; +}; + +/* + * Service Interfaces required by service_message_handler struct + */ +static void quorum_confchg_fn ( + enum totem_configuration_type configuration_type, + unsigned int *member_list, int member_list_entries, + unsigned int *left_list, int left_list_entries, + unsigned int *joined_list, int joined_list_entries, + struct memb_ring_id *ring_id); + +static void quorum_deliver_fn(unsigned int nodeid, struct iovec *iovec, int iov_len, + int endian_conversion_required); + +static int quorum_exec_init_fn (struct corosync_api_v1 *corosync_api); + +static int quorum_lib_init_fn (void *conn); + +static int quorum_lib_exit_fn (void *conn); + +static void message_handler_req_exec_quorum_nodeinfo ( + void *message, + unsigned int nodeid); + +static void message_handler_req_exec_quorum_reconfigure ( + void *message, + unsigned int nodeid); + +static void message_handler_req_exec_quorum_killnode ( + void *message, + unsigned int nodeid); + + +static void message_handler_req_lib_quorum_getinfo (void *conn, void *message); + +static void message_handler_req_lib_quorum_setexpected (void *conn, void *message); + +static void message_handler_req_lib_quorum_setvotes (void *conn, void *message); + +static void message_handler_req_lib_quorum_qdisk_register (void *conn, void *message); + +static void message_handler_req_lib_quorum_qdisk_unregister (void *conn, void *message); + +static void message_handler_req_lib_quorum_qdisk_poll (void *conn, void *message); + +static void message_handler_req_lib_quorum_qdisk_getinfo (void *conn, void *message); + +static void message_handler_req_lib_quorum_setdirty (void *conn, void *message); + +static void message_handler_req_lib_quorum_killnode (void *conn, void *message); + +static void message_handler_req_lib_quorum_leaving (void *conn, void *message); +static void message_handler_req_lib_quorum_trackstart (void *conn, void *msg); +static void message_handler_req_lib_quorum_trackstop (void *conn, void *msg); + +static void quorum_sync_init (void); +static int quorum_sync_process (void); +static void quorum_sync_activate (void); +static void quorum_sync_abort (void); + +static int quorum_exec_send_nodeinfo(void); +static int quorum_exec_send_reconfigure(int param, int nodeid, int value); +static int quorum_exec_send_killnode(int nodeid, unsigned int reason); + +/* Internal quorum API functions */ +static int quorum_api_get_quorum(void); + +/* + * Library Handler Definition + */ +static struct corosync_lib_handler quorum_lib_service[] = +{ + { /* 0 */ + .lib_handler_fn = message_handler_req_lib_quorum_getinfo, + .response_size = sizeof (struct res_lib_quorum_getinfo), + .response_id = MESSAGE_RES_QUORUM_GETINFO, + .flow_control = COROSYNC_LIB_FLOW_CONTROL_NOT_REQUIRED + }, + { /* 1 */ + .lib_handler_fn = message_handler_req_lib_quorum_setexpected, + .response_size = sizeof (struct res_lib_quorum_status), + .response_id = MESSAGE_RES_QUORUM_STATUS, + .flow_control = COROSYNC_LIB_FLOW_CONTROL_NOT_REQUIRED + }, + { /* 2 */ + .lib_handler_fn = message_handler_req_lib_quorum_setvotes, + .response_size = sizeof (struct res_lib_quorum_status), + .response_id = MESSAGE_RES_QUORUM_STATUS, + .flow_control = COROSYNC_LIB_FLOW_CONTROL_NOT_REQUIRED + }, + { /* 3 */ + .lib_handler_fn = message_handler_req_lib_quorum_qdisk_register, + .response_size = sizeof (struct res_lib_quorum_status), + .response_id = MESSAGE_RES_QUORUM_STATUS, + .flow_control = COROSYNC_LIB_FLOW_CONTROL_NOT_REQUIRED + }, + { /* 4 */ + .lib_handler_fn = message_handler_req_lib_quorum_qdisk_unregister, + .response_size = sizeof (struct res_lib_quorum_status), + .response_id = MESSAGE_RES_QUORUM_STATUS, + .flow_control = COROSYNC_LIB_FLOW_CONTROL_NOT_REQUIRED + }, + { /* 5 */ + .lib_handler_fn = message_handler_req_lib_quorum_qdisk_poll, + .response_size = sizeof (struct res_lib_quorum_status), + .response_id = MESSAGE_RES_QUORUM_STATUS, + .flow_control = COROSYNC_LIB_FLOW_CONTROL_NOT_REQUIRED + }, + { /* 6 */ + .lib_handler_fn = message_handler_req_lib_quorum_qdisk_getinfo, + .response_size = sizeof (struct res_lib_quorum_qdisk_getinfo), + .response_id = MESSAGE_RES_QUORUM_QDISK_GETINFO, + .flow_control = COROSYNC_LIB_FLOW_CONTROL_NOT_REQUIRED + }, + { /* 7 */ + .lib_handler_fn = message_handler_req_lib_quorum_setdirty, + .response_size = sizeof (struct res_lib_quorum_status), + .response_id = MESSAGE_RES_QUORUM_STATUS, + .flow_control = COROSYNC_LIB_FLOW_CONTROL_NOT_REQUIRED + }, + { /* 8 */ + .lib_handler_fn = message_handler_req_lib_quorum_killnode, + .response_size = sizeof (struct res_lib_quorum_status), + .response_id = MESSAGE_RES_QUORUM_STATUS, + .flow_control = COROSYNC_LIB_FLOW_CONTROL_NOT_REQUIRED + }, + { /* 9 */ + .lib_handler_fn = message_handler_req_lib_quorum_leaving, + .response_size = sizeof (struct res_lib_quorum_status), + .response_id = MESSAGE_RES_QUORUM_STATUS, + .flow_control = COROSYNC_LIB_FLOW_CONTROL_NOT_REQUIRED + }, + { /* 10 */ + .lib_handler_fn = message_handler_req_lib_quorum_trackstart, + .response_size = sizeof (struct res_lib_quorum_status), + .response_id = MESSAGE_RES_QUORUM_STATUS, + .flow_control = COROSYNC_LIB_FLOW_CONTROL_NOT_REQUIRED + }, + { /* 11 */ + .lib_handler_fn = message_handler_req_lib_quorum_trackstop, + .response_size = sizeof (struct res_lib_quorum_status), + .response_id = MESSAGE_RES_QUORUM_STATUS, + .flow_control = COROSYNC_LIB_FLOW_CONTROL_NOT_REQUIRED + } +}; + +static struct corosync_service_engine quorum_service_handler = { + .name = "corosync cluster quorum service v1.01", + .id = QUORUM_SERVICE, + .private_data_size = sizeof (struct quorum_pd), + .flow_control = COROSYNC_LIB_FLOW_CONTROL_REQUIRED, + .lib_init_fn = quorum_lib_init_fn, + .lib_exit_fn = quorum_lib_exit_fn, + .lib_engine = quorum_lib_service, + .lib_engine_count = sizeof (quorum_lib_service) / sizeof (struct corosync_lib_handler), + .exec_init_fn = quorum_exec_init_fn, + .exec_engine = NULL, + .exec_engine_count = 0, + .confchg_fn = NULL, /* Invoked by tpg */ + .sync_init = quorum_sync_init, + .sync_process = quorum_sync_process, + .sync_activate = quorum_sync_activate, + .sync_abort = quorum_sync_abort +}; + +/* + * Dynamic loader definition + */ +static struct corosync_service_engine *quorum_get_service_handler_ver0 (void); + +static struct corosync_service_engine_iface_ver0 quorum_service_handler_iface = { + .corosync_get_service_engine_ver0 = quorum_get_service_handler_ver0 +}; + +static struct quorum_services_api_ver1 quorum_service_api_v1 = { + .quorum_api_get_quorum = quorum_api_get_quorum +}; + +static struct lcr_iface corosync_quorum_ver0[2] = { + { + .name = "corosync_quorum", + .version = 0, + .versions_replace = 0, + .versions_replace_count = 0, + .dependencies = 0, + .dependency_count = 0, + .constructor = NULL, + .destructor = NULL, + .interfaces = NULL + }, + { + .name = "corosync_quorum_api", + .version = 0, + .versions_replace = 0, + .versions_replace_count = 0, + .dependencies = 0, + .dependency_count = 0, + .constructor = NULL, + .destructor = NULL, + .interfaces = NULL + } +}; + +static struct lcr_comp quorum_comp_ver0 = { + .iface_count = 2, + .ifaces = corosync_quorum_ver0 +}; + + +static struct corosync_service_engine *quorum_get_service_handler_ver0 (void) +{ + return (&quorum_service_handler); +} + +__attribute__ ((constructor)) static void quorum_comp_register (void) { + lcr_interfaces_set (&corosync_quorum_ver0[0], &quorum_service_handler_iface); + lcr_interfaces_set (&corosync_quorum_ver0[1], &quorum_service_api_v1); + + lcr_component_register (&quorum_comp_ver0); +} +#define QUORUM_MSG_ACK 1 +#define QUORUM_MSG_PORTOPENED 2 +#define QUORUM_MSG_PORTCLOSED 3 +#define QUORUM_MSG_BARRIER 4 +#define QUORUM_MSG_NODEINFO 5 +#define QUORUM_MSG_KILLNODE 6 +#define QUORUM_MSG_LEAVE 7 +#define QUORUM_MSG_RECONFIGURE 8 +#define QUORUM_MSG_PORTENQ 9 +#define QUORUM_MSG_PORTSTATUS 10 +#define QUORUM_MSG_FENCESTATUS 11 + +struct req_exec_quorum_nodeinfo { + unsigned char cmd; + unsigned char first_trans; + uint16_t cluster_id; + int high_nodeid; + int expected_votes; + + unsigned int major_version; /* Not backwards compatible */ + unsigned int minor_version; /* Backwards compatible */ + unsigned int patch_version; /* Backwards/forwards compatible */ + unsigned int config_version; + unsigned int flags; + uint64_t fence_time; /* not used */ + uint64_t join_time; + char clustername[16]; /* not used */ + char fence_agent[]; /* not used */ +}; + +/* Parameters for RECONFIG command */ +#define RECONFIG_PARAM_EXPECTED_VOTES 1 +#define RECONFIG_PARAM_NODE_VOTES 2 +#define RECONFIG_PARAM_LEAVING 3 + +struct req_exec_quorum_reconfigure { + unsigned char cmd; + unsigned char param; + unsigned short pad; + int nodeid; + unsigned int value; +}; + +struct req_exec_quorum_killnode { + unsigned char cmd; + unsigned char pad1; + uint16_t reason; + int nodeid; +}; + + +/* These just make the access a little neater */ +static inline int objdb_get_string(struct corosync_api_v1 *corosync, unsigned int object_service_handle, + char *key, char **value) +{ + int res; + + *value = NULL; + if ( !(res = corosync->object_key_get(object_service_handle, + key, + strlen(key), + (void *)value, + NULL))) { + if (*value) + return 0; + } + return -1; +} + +static inline void objdb_get_int(struct corosync_api_v1 *corosync, unsigned int object_service_handle, + char *key, unsigned int *intvalue, unsigned int default_value) +{ + char *value = NULL; + + *intvalue = default_value; + + if (!corosync->object_key_get(object_service_handle, key, strlen(key), + (void *)&value, NULL)) { + if (value) { + *intvalue = atoi(value); + } + } +} + +static int quorum_send_message(void *message, int len) +{ + struct iovec iov[2]; + struct q_protheader header; + + header.tgtport = 0; + header.srcport = 0; + header.flags = 0; + header.srcid = us->node_id; + header.tgtid = 0; + + iov[0].iov_base = &header; + iov[0].iov_len = sizeof(header); + iov[1].iov_base = message; + iov[1].iov_len = len; + + return corosync_api->tpg_joined_mcast(group_handle, iov, 2, TOTEM_AGREED); +} + +static void quorum_sync_init (void) +{ +} + +static int quorum_sync_process (void) +{ + return quorum_exec_send_nodeinfo(); +} + +static void quorum_sync_activate (void) +{ + +} +static void quorum_sync_abort (void) +{ + +} + +static int quorum_exec_init_fn (struct corosync_api_v1 *api) +{ + unsigned int object_handle; + unsigned int find_handle; + + log_printf(LOG_LEVEL_NOTICE, "quorum_exec_init_fn \n"); + + corosync_api = api; + + list_init(&cluster_members_list); + list_init(&trackers_list); + + /* Allocate a cluster_node for us */ + us = allocate_node(corosync_api->totem_nodeid_get()); + if (!us) + return (1); + + us->flags |= NODE_FLAGS_US; + us->state = NODESTATE_MEMBER; + us->expected_votes = DEFAULT_EXPECTED; + us->votes = 1; + + /* Get configuration variables */ + corosync_api->object_find_create(OBJECT_PARENT_HANDLE, "quorum", strlen("quorum"), &find_handle); + + if (corosync_api->object_find_next(find_handle, &object_handle) == 0) { + unsigned int value = 0; + objdb_get_int(corosync_api, object_handle, "expected_votes", &us->expected_votes, DEFAULT_EXPECTED); + objdb_get_int(corosync_api, object_handle, "votes", &us->votes, 1); + objdb_get_int(corosync_api, object_handle, "two_node", &two_node, 0); + objdb_get_int(corosync_api, object_handle, "quorumdev_poll", &quorumdev_poll, DEFAULT_QDEV_POLL); + objdb_get_int(corosync_api, object_handle, "disallowed", &value, 0); + if (value) + quorum_flags |= QUORUM_FLAG_FEATURE_DISALLOWED; + } + corosync_api->object_find_destroy(find_handle); + + api->tpg_init(&group_handle, quorum_deliver_fn, quorum_confchg_fn); + api->tpg_join(group_handle, quorum_group, 1); + return (0); +} + +static int quorum_lib_exit_fn (void *conn) +{ + struct quorum_pd *quorum_pd = (struct quorum_pd *)corosync_api->ipc_private_data_get (conn); + + if (quorum_pd->tracking_enabled) { + list_del (&quorum_pd->list); + list_init (&quorum_pd->list); + } + return (0); +} + + +static int send_quorum_notification(void *conn) +{ + struct res_lib_quorum_notification *res_lib_quorum_notification; + struct list_head *tmp; + struct cluster_node *node; + int cluster_members = 0; + int i = 0; + int size; + char *buf; + + list_iterate(tmp, &cluster_members_list) { + node = list_entry(tmp, struct cluster_node, list); + cluster_members++; + } + if (quorum_device) + cluster_members++; + + size = sizeof(struct res_lib_quorum_notification) + sizeof(struct quorum_node) * cluster_members; + buf = alloca(size); + if (!buf) + return -1; + + res_lib_quorum_notification = (struct res_lib_quorum_notification *)buf; + res_lib_quorum_notification->quorate = cluster_is_quorate; + res_lib_quorum_notification->node_list_entries = cluster_members; + list_iterate(tmp, &cluster_members_list) { + node = list_entry(tmp, struct cluster_node, list); + res_lib_quorum_notification->node_list[i].nodeid = node->node_id; + res_lib_quorum_notification->node_list[i++].state = node->state; + } + if (quorum_device) { + res_lib_quorum_notification->node_list[i].nodeid = 0; + res_lib_quorum_notification->node_list[i++].state = quorum_device->state | 0x80; + } + res_lib_quorum_notification->header.id = MESSAGE_RES_QUORUM_NOTIFICATION; + res_lib_quorum_notification->header.size = size; + res_lib_quorum_notification->header.error = SA_AIS_OK; + + /* Send it to all interested parties */ + if (conn) { + return corosync_api->ipc_conn_send_response(conn, buf, size); + } + else { + struct quorum_pd *qpd; + + list_iterate(tmp, &trackers_list) { + qpd = list_entry(tmp, struct quorum_pd, list); + + corosync_api->ipc_conn_send_response(corosync_api->ipc_conn_partner_get(qpd->conn), buf, size); + } + } + return 0; +} + +/* If "cluster_is_quorate" is 0 then all activity apart from protected ports is + * blocked. */ +static void set_quorate(int total_votes) +{ + int quorate; + + if (quorum > total_votes) { + quorate = 0; + } + else { + quorate = 1; + } + + if (cluster_is_quorate && !quorate) + log_printf(LOG_INFO, "quorum lost, blocking activity\n"); + if (!cluster_is_quorate && quorate) + log_printf(LOG_INFO, "quorum regained, resuming activity\n"); + + /* If we are newly quorate, then kill any AISONLY nodes */ + if (!cluster_is_quorate && quorate) { + struct cluster_node *node = NULL; + struct list_head *tmp; + + list_iterate(tmp, &cluster_members_list) { + node = list_entry(tmp, struct cluster_node, list); + if (node->state == NODESTATE_AISONLY) + quorum_exec_send_killnode(node->node_id, QUORUM_REASON_KILL_REJOIN); + } + } + + cluster_is_quorate = quorate; +} + +static int calculate_quorum(int allow_decrease, unsigned int *ret_total_votes) +{ + struct list_head *nodelist; + struct cluster_node *node; + unsigned int total_votes = 0; + unsigned int highest_expected = 0; + unsigned int newquorum, q1, q2; + unsigned int total_nodes = 0; + unsigned int max_expected = 0; + unsigned int leaving = 0; + + list_iterate(nodelist, &cluster_members_list) { + node = list_entry(nodelist, struct cluster_node, list); + + if (node->state == NODESTATE_MEMBER) { + highest_expected = + max(highest_expected, node->expected_votes); + total_votes += node->votes; + total_nodes++; + } + if (node->state == NODESTATE_LEAVING) { + leaving = 1; + } + } + + if (quorum_device && quorum_device->state == NODESTATE_MEMBER) + total_votes += quorum_device->votes; + + if (max_expected > 0) + highest_expected = max_expected; + + /* This quorum calculation is taken from the OpenVMS Cluster Systems + * manual, but, then, you guessed that didn't you */ + q1 = (highest_expected + 2) / 2; + q2 = (total_votes + 2) / 2; + newquorum = max(q1, q2); + + /* Normally quorum never decreases but the system administrator can + * force it down by setting expected votes to a maximum value */ + if (!allow_decrease) + newquorum = max(quorum, newquorum); + + /* The special two_node mode allows each of the two nodes to retain + * quorum if the other fails. Only one of the two should live past + * fencing (as both nodes try to fence each other in split-brain.) + * Also: if there are more than two nodes, force us inquorate to avoid + * any damage or confusion. + */ + if (two_node && total_nodes <= 2) + newquorum = 1; + + if (ret_total_votes) + *ret_total_votes = total_votes; + return newquorum; +} + +/* Recalculate cluster quorum, set quorate and notify changes */ +static void recalculate_quorum(int allow_decrease) +{ + unsigned int total_votes; + + quorum = calculate_quorum(allow_decrease, &total_votes); + set_quorate(total_votes); + send_quorum_notification(NULL); +} + +static int have_disallowed(void) +{ + struct cluster_node *node; + struct list_head *tmp; + + list_iterate(tmp, &cluster_members_list) { + node = list_entry(tmp, struct cluster_node, list); + if (node->state == NODESTATE_AISONLY) + return 1; + } + + return 0; +} + +static void node_add_ordered(struct cluster_node *newnode) +{ + struct cluster_node *node = NULL; + struct list_head *tmp; + struct list_head *newlist = &newnode->list; + + list_iterate(tmp, &cluster_members_list) { + node = list_entry(tmp, struct cluster_node, list); + + if (newnode->node_id < node->node_id) + break; + } + + if (!node) + list_add(&newnode->list, &cluster_members_list); + else { + newlist->prev = tmp->prev; + newlist->next = tmp; + tmp->prev->next = newlist; + tmp->prev = newlist; + } +} + +static struct cluster_node *allocate_node(int nodeid) +{ + struct cluster_node *cl; + + cl = malloc(sizeof(struct cluster_node)); + if (cl) { + memset(cl, 0, sizeof(struct cluster_node)); + cl->node_id = nodeid; + if (nodeid) + node_add_ordered(cl); + } + return cl; +} + +static struct cluster_node *find_node_by_nodeid(int nodeid) +{ + struct cluster_node *node; + struct list_head *tmp; + + list_iterate(tmp, &cluster_members_list) { + node = list_entry(tmp, struct cluster_node, list); + if (node->node_id == nodeid) + return node; + } + return NULL; +} + + +static int quorum_exec_send_nodeinfo() +{ + struct req_exec_quorum_nodeinfo req_exec_quorum_nodeinfo; + + log_printf(LOG_LEVEL_DEBUG, "Sending nodeinfo message"); + + req_exec_quorum_nodeinfo.cmd = QUORUM_MSG_NODEINFO; + req_exec_quorum_nodeinfo.expected_votes = us->expected_votes; +// req_exec_quorum_nodeinfo.votes = us->votes; + req_exec_quorum_nodeinfo.major_version = QUORUM_MAJOR_VERSION; + req_exec_quorum_nodeinfo.minor_version = QUORUM_MINOR_VERSION; + req_exec_quorum_nodeinfo.patch_version = QUORUM_PATCH_VERSION; + req_exec_quorum_nodeinfo.flags = us->flags; + req_exec_quorum_nodeinfo.first_trans = first_trans; // TODO Check this works from sync_process() + if (have_disallowed()) + req_exec_quorum_nodeinfo.flags |= NODE_FLAGS_SEESDISALLOWED; + + return quorum_send_message(&req_exec_quorum_nodeinfo, sizeof(req_exec_quorum_nodeinfo)); +} + + +static int quorum_exec_send_reconfigure(int param, int nodeid, int value) +{ + struct req_exec_quorum_reconfigure req_exec_quorum_reconfigure; + + log_printf(LOG_LEVEL_DEBUG, "Sending reconfigure message"); + + req_exec_quorum_reconfigure.cmd = QUORUM_MSG_RECONFIGURE; + req_exec_quorum_reconfigure.param = param; + req_exec_quorum_reconfigure.nodeid = nodeid; + req_exec_quorum_reconfigure.value = value; + + return quorum_send_message(&req_exec_quorum_reconfigure, sizeof(req_exec_quorum_reconfigure)); +} + +static int quorum_exec_send_killnode(int nodeid, unsigned int reason) +{ + struct req_exec_quorum_killnode req_exec_quorum_killnode; + + log_printf(LOG_LEVEL_DEBUG, "Sending killnode message"); + + req_exec_quorum_killnode.cmd = QUORUM_MSG_KILLNODE; + req_exec_quorum_killnode.nodeid = nodeid; + req_exec_quorum_killnode.reason = reason; + + + return quorum_send_message(&req_exec_quorum_killnode, sizeof(req_exec_quorum_killnode)); +} + +static void quorum_confchg_fn ( + enum totem_configuration_type configuration_type, + unsigned int *member_list, int member_list_entries, + unsigned int *left_list, int left_list_entries, + unsigned int *joined_list, int joined_list_entries, + struct memb_ring_id *ring_id) +{ + int i; + int leaving = 0; + struct cluster_node *node; + + if (member_list_entries > 1) + first_trans = 0; + + if (left_list_entries) { + for (i = 0; i< left_list_entries; i++) { + node = find_node_by_nodeid(left_list[i]); + if (node) { + if (node->state == NODESTATE_LEAVING) + leaving = 1; + node->state = NODESTATE_DEAD; + node->flags |= NODE_FLAGS_BEENDOWN; + } + } + recalculate_quorum(leaving); + } +} + +/* TODO: Byteswap messages */ +static void exec_quorum_nodeinfo_endian_convert (void *msg) +{ +} + +static void exec_quorum_reconfigure_endian_convert (void *msg) +{ +} + +static void exec_quorum_killnode_endian_convert (void *msg) +{ +} + +static void quorum_deliver_fn(unsigned int nodeid, struct iovec *iovec, int iov_len, + int endian_conversion_required) +{ + struct q_protheader *header = iovec->iov_base; + char *buf; + + if (endian_conversion_required) { + header->srcid = swab32(header->srcid); + header->tgtid = swab32(header->tgtid); + header->flags = swab32(header->flags); + } + + /* Only pass on messages for us or everyone */ + if (header->tgtport == 0 && + (header->tgtid == us->node_id || + header->tgtid == 0)) { + buf = iovec->iov_base + sizeof(struct q_protheader), iovec->iov_len - sizeof(struct q_protheader); + switch (*buf) { + + case QUORUM_MSG_NODEINFO: + if (endian_conversion_required) + exec_quorum_nodeinfo_endian_convert(buf); + message_handler_req_exec_quorum_nodeinfo (buf, header->srcid); + break; + case QUORUM_MSG_RECONFIGURE: + if (endian_conversion_required) + exec_quorum_reconfigure_endian_convert(buf); + message_handler_req_exec_quorum_reconfigure (buf, header->srcid); + break; + case QUORUM_MSG_KILLNODE: + if (endian_conversion_required) + exec_quorum_killnode_endian_convert(buf); + message_handler_req_exec_quorum_killnode (buf, header->srcid); + break; + + /* Just ignore other messages */ + } + } +} + +static void message_handler_req_exec_quorum_nodeinfo ( + void *message, + unsigned int nodeid) +{ + struct req_exec_quorum_nodeinfo *req_exec_quorum_nodeinfo = (struct req_exec_quorum_nodeinfo *)message; + struct cluster_node *node; + + log_printf(LOG_LEVEL_DEBUG, "got nodeinfo message from cluster node %d\n", nodeid); + + node = find_node_by_nodeid(nodeid); + if (!node) { + node = allocate_node(nodeid); + } + if (!node) { + // TODO enomem error + return; + } + + if (req_exec_quorum_nodeinfo->flags & NODE_FLAGS_SEESDISALLOWED && !have_disallowed()) { + /* Must use syslog directly here or the message will never arrive */ + syslog(LOG_CRIT, "[QUORUM]: Joined a cluster with disallowed nodes. must die"); + exit(2); + } + + /* Update node state */ +// node->votes = req_exec_quorum_nodeinfo->votes; + node->expected_votes = req_exec_quorum_nodeinfo->expected_votes; + node->state = NODESTATE_MEMBER; + + /* Check flags for disallowed (if enabled) */ + if (quorum_flags & QUORUM_FLAG_FEATURE_DISALLOWED) { + if ((req_exec_quorum_nodeinfo->flags & NODE_FLAGS_DIRTY && node->flags & NODE_FLAGS_BEENDOWN) || + (req_exec_quorum_nodeinfo->flags & NODE_FLAGS_DIRTY && req_exec_quorum_nodeinfo->first_trans && !(node->flags & NODE_FLAGS_US))) { + if (node->state != NODESTATE_AISONLY) { + if (cluster_is_quorate) { + log_printf(LOG_CRIT, "Killing node %d because it has rejoined the cluster with existing state", node->node_id); + node->state = NODESTATE_AISONLY; + quorum_exec_send_killnode(nodeid, QUORUM_REASON_KILL_REJOIN); + } + else { + log_printf(LOG_CRIT, "Node %d not joined to quorum because it has existing state", node->node_id); + node->state = NODESTATE_AISONLY; + } + } + } + } + node->flags &= ~NODE_FLAGS_BEENDOWN; + + // TODO do we need this as well as in confchg ? + recalculate_quorum(0); +} + +static void message_handler_req_exec_quorum_killnode ( + void *message, + unsigned int nodeid) +{ + struct req_exec_quorum_killnode *req_exec_quorum_killnode = (struct req_exec_quorum_killnode *)message; + + if (req_exec_quorum_killnode->nodeid == corosync_api->totem_nodeid_get()) { + log_printf(LOG_CRIT, "Killed by node %d: %s\n", nodeid, kill_reason(req_exec_quorum_killnode->reason)); + + // Is there a better way!! ???? + exit(1); + } +} + +static void message_handler_req_exec_quorum_reconfigure ( + void *message, + unsigned int nodeid) +{ + struct req_exec_quorum_reconfigure *req_exec_quorum_reconfigure = (struct req_exec_quorum_reconfigure *)message; + struct cluster_node *node; + struct list_head *nodelist; + + log_printf(LOG_LEVEL_DEBUG, "got reconfigure message from cluster node %d\n", nodeid); + + node = find_node_by_nodeid(req_exec_quorum_reconfigure->nodeid); + if (!node) + return; + + switch(req_exec_quorum_reconfigure->param) + { + case RECONFIG_PARAM_EXPECTED_VOTES: + node->expected_votes = req_exec_quorum_reconfigure->value; + + list_iterate(nodelist, &cluster_members_list) { + node = list_entry(nodelist, struct cluster_node, list); + if (node->state == NODESTATE_MEMBER && + node->expected_votes > req_exec_quorum_reconfigure->value) { + node->expected_votes = req_exec_quorum_reconfigure->value; + } + } + recalculate_quorum(1); /* Allow decrease */ + break; + + case RECONFIG_PARAM_NODE_VOTES: + node->votes = req_exec_quorum_reconfigure->value; + recalculate_quorum(1); /* Allow decrease */ + break; + + case RECONFIG_PARAM_LEAVING: + node->state = NODESTATE_LEAVING; + break; + } +} + +static int quorum_lib_init_fn (void *conn) +{ + struct quorum_pd *pd = (struct quorum_pd *)corosync_api->ipc_private_data_get (conn); + + log_printf(LOG_LEVEL_DEBUG, "lib_init_fn: conn=%p\n", conn); + + list_init (&pd->list); + pd->conn = conn; + + return (0); +} + +/* Message from the library */ +static void message_handler_req_lib_quorum_getinfo (void *conn, void *message) +{ + struct req_lib_quorum_getinfo *req_lib_quorum_getinfo = (struct req_lib_quorum_getinfo *)message; + struct res_lib_quorum_getinfo res_lib_quorum_getinfo; + struct cluster_node *node; + int highest_expected = 0; + int total_votes = 0; + SaAisErrorT error = SA_AIS_OK; + + log_printf(LOG_LEVEL_DEBUG, "got getinfo request on %p for node %d\n", conn, req_lib_quorum_getinfo->nodeid); + + if (req_lib_quorum_getinfo->nodeid) { + node = find_node_by_nodeid(req_lib_quorum_getinfo->nodeid); + } + else { + node = us; + } + + if (node) { + struct cluster_node *iternode; + struct list_head *nodelist; + + list_iterate(nodelist, &cluster_members_list) { + iternode = list_entry(nodelist, struct cluster_node, list); + + if (node->state == NODESTATE_MEMBER) { + highest_expected = + max(highest_expected, node->expected_votes); + total_votes += node->votes; + } + } + + if (quorum_device && quorum_device->state == NODESTATE_MEMBER) { + total_votes += quorum_device->votes; + } + + res_lib_quorum_getinfo.votes = us->votes; + res_lib_quorum_getinfo.expected_votes = us->expected_votes; + res_lib_quorum_getinfo.highest_expected = highest_expected; + + res_lib_quorum_getinfo.quorum = quorum; + res_lib_quorum_getinfo.total_votes = total_votes; + res_lib_quorum_getinfo.flags = 0; + + if (us->flags & NODE_FLAGS_DIRTY) + res_lib_quorum_getinfo.flags |= QUORUM_INFO_FLAG_DIRTY; + if (two_node) + res_lib_quorum_getinfo.flags |= QUORUM_INFO_FLAG_TWONODE; + if (cluster_is_quorate) + res_lib_quorum_getinfo.flags |= QUORUM_INFO_FLAG_QUORATE; + if (us->flags & NODE_FLAGS_SEESDISALLOWED) + res_lib_quorum_getinfo.flags |= QUORUM_INFO_FLAG_DISALLOWED; + } + else { + error = SA_AIS_ERR_NOT_EXIST; + } + + res_lib_quorum_getinfo.header.size = sizeof(res_lib_quorum_getinfo); + res_lib_quorum_getinfo.header.id = MESSAGE_RES_QUORUM_GETINFO; + res_lib_quorum_getinfo.header.error = error; + corosync_api->ipc_conn_send_response(conn, &res_lib_quorum_getinfo, sizeof(res_lib_quorum_getinfo)); + log_printf(LOG_LEVEL_DEBUG, "getinfo response error: %d\n", error); +} + +/* Message from the library */ +static void message_handler_req_lib_quorum_killnode (void *conn, void *message) +{ + struct req_lib_quorum_killnode *req_lib_quorum_killnode = (struct req_lib_quorum_killnode *)message; + struct res_lib_quorum_status res_lib_quorum_status; + SaAisErrorT error = SA_AIS_OK; + + log_printf(LOG_LEVEL_DEBUG, "got killnode request on %p\n", conn); + + quorum_exec_send_killnode(req_lib_quorum_killnode->nodeid, req_lib_quorum_killnode->reason); + + /* send status */ + res_lib_quorum_status.header.size = sizeof(res_lib_quorum_status); + res_lib_quorum_status.header.id = MESSAGE_RES_QUORUM_STATUS; + res_lib_quorum_status.header.error = error; + corosync_api->ipc_conn_send_response(conn, &res_lib_quorum_status, sizeof(res_lib_quorum_status)); +} +/* Message from the library */ +static void message_handler_req_lib_quorum_setexpected (void *conn, void *message) +{ + struct req_lib_quorum_setexpected *req_lib_quorum_setexpected = (struct req_lib_quorum_setexpected *)message; + struct res_lib_quorum_status res_lib_quorum_status; + SaAisErrorT error = SA_AIS_OK; + + log_printf(LOG_LEVEL_DEBUG, "got setexpected request on %p\n", conn); + + // TODO validate as cman does + + quorum_exec_send_reconfigure(RECONFIG_PARAM_EXPECTED_VOTES, us->node_id, req_lib_quorum_setexpected->expected_votes); + + /* send status */ + res_lib_quorum_status.header.size = sizeof(res_lib_quorum_status); + res_lib_quorum_status.header.id = MESSAGE_RES_QUORUM_STATUS; + res_lib_quorum_status.header.error = error; + corosync_api->ipc_conn_send_response(conn, &res_lib_quorum_status, sizeof(res_lib_quorum_status)); +} + +/* Message from the library */ +static void message_handler_req_lib_quorum_setvotes (void *conn, void *message) +{ + struct req_lib_quorum_setvotes *req_lib_quorum_setvotes = (struct req_lib_quorum_setvotes *)message; + struct res_lib_quorum_status res_lib_quorum_status; + SaAisErrorT error = SA_AIS_OK; + + log_printf(LOG_LEVEL_DEBUG, "got setvotes request on %p\n", conn); + + // TODO validate as cman does + + if (!req_lib_quorum_setvotes->nodeid) + req_lib_quorum_setvotes->nodeid = corosync_api->totem_nodeid_get(); + + quorum_exec_send_reconfigure(RECONFIG_PARAM_NODE_VOTES, req_lib_quorum_setvotes->nodeid, req_lib_quorum_setvotes->votes); + + /* send status */ + res_lib_quorum_status.header.size = sizeof(res_lib_quorum_status); + res_lib_quorum_status.header.id = MESSAGE_RES_QUORUM_STATUS; + res_lib_quorum_status.header.error = error; + corosync_api->ipc_conn_send_response(conn, &res_lib_quorum_status, sizeof(res_lib_quorum_status)); +} + +static void message_handler_req_lib_quorum_leaving (void *conn, void *message) +{ + struct res_lib_quorum_status res_lib_quorum_status; + SaAisErrorT error = SA_AIS_OK; + + log_printf(LOG_LEVEL_DEBUG, "got leaving request on %p\n", conn); + + quorum_exec_send_reconfigure(RECONFIG_PARAM_LEAVING, us->node_id, 0); + + /* send status */ + res_lib_quorum_status.header.size = sizeof(res_lib_quorum_status); + res_lib_quorum_status.header.id = MESSAGE_RES_QUORUM_STATUS; + res_lib_quorum_status.header.error = error; + corosync_api->ipc_conn_send_response(conn, &res_lib_quorum_status, sizeof(res_lib_quorum_status)); +} + +static void quorum_device_timer_fn(void *arg) +{ + struct timeval now; + if (!quorum_device || quorum_device->state == NODESTATE_DEAD) + return; + + log_printf(LOG_DEBUG, "quorum_device_timer_fn\n"); + gettimeofday(&now, NULL); + if (quorum_device->last_hello.tv_sec + quorumdev_poll/1000 < now.tv_sec) { + quorum_device->state = NODESTATE_DEAD; + log_printf(LOG_INFO, "lost contact with quorum device\n"); + recalculate_quorum(0); + } + else { + corosync_api->timer_add_duration((unsigned long long)quorumdev_poll*1000000, quorum_device, + quorum_device_timer_fn, &quorum_device_timer); + } +} + + +static void message_handler_req_lib_quorum_qdisk_register (void *conn, void *message) +{ + struct req_lib_quorum_qdisk_register *req_lib_quorum_qdisk_register = (struct req_lib_quorum_qdisk_register *)message; + struct res_lib_quorum_status res_lib_quorum_status; + SaAisErrorT error = SA_AIS_OK; + + log_printf(LOG_LEVEL_DEBUG, "got qdisk_register request on %p\n", conn); + + + if (quorum_device) { + error = SA_AIS_ERR_EXIST; + } + else { + quorum_device = allocate_node(0); + quorum_device->state = NODESTATE_DEAD; + quorum_device->votes = req_lib_quorum_qdisk_register->votes; + strcpy(quorum_device_name, req_lib_quorum_qdisk_register->name); + list_add(&quorum_device->list, &cluster_members_list); + } + + /* send status */ + res_lib_quorum_status.header.size = sizeof(res_lib_quorum_status); + res_lib_quorum_status.header.id = MESSAGE_RES_QUORUM_STATUS; + res_lib_quorum_status.header.error = error; + corosync_api->ipc_conn_send_response(conn, &res_lib_quorum_status, sizeof(res_lib_quorum_status)); +} + +static void message_handler_req_lib_quorum_qdisk_unregister (void *conn, void *message) +{ + struct res_lib_quorum_status res_lib_quorum_status; + SaAisErrorT error = SA_AIS_OK; + + log_printf(LOG_LEVEL_DEBUG, "got qdisk_unregister request on %p\n", conn); + + if (quorum_device) { + struct cluster_node *node = quorum_device; + + quorum_device = NULL; + list_del(&node->list); + free(node); + recalculate_quorum(0); + } + else { + error = SA_AIS_ERR_NOT_EXIST; + } + + /* send status */ + res_lib_quorum_status.header.size = sizeof(res_lib_quorum_status); + res_lib_quorum_status.header.id = MESSAGE_RES_QUORUM_STATUS; + res_lib_quorum_status.header.error = error; + corosync_api->ipc_conn_send_response(conn, &res_lib_quorum_status, sizeof(res_lib_quorum_status)); +} + +static void message_handler_req_lib_quorum_qdisk_poll (void *conn, void *message) +{ + struct req_lib_quorum_qdisk_poll *req_lib_quorum_qdisk_poll = (struct req_lib_quorum_qdisk_poll *)message; + struct res_lib_quorum_status res_lib_quorum_status; + SaAisErrorT error = SA_AIS_OK; + + log_printf(LOG_LEVEL_DEBUG, "got qdisk_poll request on %p, state=%d\n", conn, req_lib_quorum_qdisk_poll->state); + + if (quorum_device) { + if (req_lib_quorum_qdisk_poll->state) { + gettimeofday(&quorum_device->last_hello, NULL); + if (quorum_device->state == NODESTATE_DEAD) { + quorum_device->state = NODESTATE_MEMBER; + recalculate_quorum(0); + + corosync_api->timer_add_duration((unsigned long long)quorumdev_poll*1000000, quorum_device, + quorum_device_timer_fn, &quorum_device_timer); + } + } + else { + if (quorum_device->state == NODESTATE_MEMBER) { + quorum_device->state = NODESTATE_DEAD; + recalculate_quorum(0); + corosync_api->timer_delete(quorum_device_timer); + } + } + } + else { + error = SA_AIS_ERR_NOT_EXIST; + } + + /* send status */ + res_lib_quorum_status.header.size = sizeof(res_lib_quorum_status); + res_lib_quorum_status.header.id = MESSAGE_RES_QUORUM_STATUS; + res_lib_quorum_status.header.error = error; + corosync_api->ipc_conn_send_response(conn, &res_lib_quorum_status, sizeof(res_lib_quorum_status)); +} + +static void message_handler_req_lib_quorum_qdisk_getinfo (void *conn, void *message) +{ + struct res_lib_quorum_qdisk_getinfo res_lib_quorum_qdisk_getinfo; + SaAisErrorT error = SA_AIS_OK; + + log_printf(LOG_LEVEL_DEBUG, "got qdisk_getinfo on %p\n", conn); + + if (quorum_device) { + log_printf(LOG_LEVEL_DEBUG, "got qdisk_getinfo state %d\n", quorum_device->state); + res_lib_quorum_qdisk_getinfo.votes = quorum_device->votes; + if (quorum_device->state == NODESTATE_MEMBER) + res_lib_quorum_qdisk_getinfo.state = 1; + else + res_lib_quorum_qdisk_getinfo.state = 0; + strcpy(res_lib_quorum_qdisk_getinfo.name, quorum_device_name); + } + else { + error = SA_AIS_ERR_NOT_EXIST; + } + + /* send status */ + res_lib_quorum_qdisk_getinfo.header.size = sizeof(res_lib_quorum_qdisk_getinfo); + res_lib_quorum_qdisk_getinfo.header.id = MESSAGE_RES_QUORUM_GETINFO; + res_lib_quorum_qdisk_getinfo.header.error = error; + corosync_api->ipc_conn_send_response(conn, &res_lib_quorum_qdisk_getinfo, sizeof(res_lib_quorum_qdisk_getinfo)); +} + +static void message_handler_req_lib_quorum_setdirty (void *conn, void *message) +{ + struct res_lib_quorum_status res_lib_quorum_status; + SaAisErrorT error = SA_AIS_OK; + + log_printf(LOG_LEVEL_DEBUG, "got setdirty request on %p\n", conn); + + us->flags |= NODE_FLAGS_DIRTY; + + /* send status */ + res_lib_quorum_status.header.size = sizeof(res_lib_quorum_status); + res_lib_quorum_status.header.id = MESSAGE_RES_QUORUM_STATUS; + res_lib_quorum_status.header.error = error; + corosync_api->ipc_conn_send_response(conn, &res_lib_quorum_status, sizeof(res_lib_quorum_status)); +} + +static void message_handler_req_lib_quorum_trackstart (void *conn, void *msg) +{ + struct req_lib_quorum_trackstart *req_lib_quorum_trackstart = (struct req_lib_quorum_trackstart *)msg; + struct res_lib_quorum_status res_lib_quorum_status; + struct quorum_pd *quorum_pd = (struct quorum_pd *)corosync_api->ipc_private_data_get (conn); + + log_printf(LOG_LEVEL_DEBUG, "got trackstart request on %p\n", conn); + /* + * If an immediate listing of the current cluster membership + * is requested, generate membership list + */ + if (req_lib_quorum_trackstart->track_flags & SA_TRACK_CURRENT || + req_lib_quorum_trackstart->track_flags & SA_TRACK_CHANGES) { + log_printf(LOG_LEVEL_DEBUG, "sending initial status to %p\n", conn); + send_quorum_notification(corosync_api->ipc_conn_partner_get (conn)); + } + + /* + * Record requests for tracking + */ + if (req_lib_quorum_trackstart->track_flags & SA_TRACK_CHANGES || + req_lib_quorum_trackstart->track_flags & SA_TRACK_CHANGES_ONLY) { + + quorum_pd->track_flags = req_lib_quorum_trackstart->track_flags; + quorum_pd->tracking_enabled = 1; + + list_add (&quorum_pd->list, &trackers_list); + } + + /* send status */ + res_lib_quorum_status.header.size = sizeof(res_lib_quorum_status); + res_lib_quorum_status.header.id = MESSAGE_RES_QUORUM_STATUS; + res_lib_quorum_status.header.error = SA_AIS_OK; + corosync_api->ipc_conn_send_response(conn, &res_lib_quorum_status, sizeof(res_lib_quorum_status)); +} + +static void message_handler_req_lib_quorum_trackstop (void *conn, void *msg) +{ + struct res_lib_quorum_status res_lib_quorum_status; + struct quorum_pd *quorum_pd = (struct quorum_pd *)corosync_api->ipc_private_data_get (conn); + + log_printf(LOG_LEVEL_DEBUG, "got trackstop request on %p\n", conn); + + if (quorum_pd->tracking_enabled) { + res_lib_quorum_status.header.error = SA_AIS_OK; + quorum_pd->tracking_enabled = 0; + list_del (&quorum_pd->list); + list_init (&quorum_pd->list); + } else { + res_lib_quorum_status.header.error = SA_AIS_ERR_NOT_EXIST; + } + + + /* send status */ + res_lib_quorum_status.header.size = sizeof(res_lib_quorum_status); + res_lib_quorum_status.header.id = MESSAGE_RES_QUORUM_STATUS; + res_lib_quorum_status.header.error = SA_AIS_OK; + corosync_api->ipc_conn_send_response(conn, &res_lib_quorum_status, sizeof(res_lib_quorum_status)); +} + + +static char *kill_reason(int reason) +{ + static char msg[1024]; + + switch (reason) + { + case QUORUM_REASON_KILL_REJECTED: + return "our membership application was rejected"; + + case QUORUM_REASON_KILL_APPLICATION: + return "we were killed by an application request"; + + case QUORUM_REASON_KILL_REJOIN: + return "we rejoined the cluster without a full restart"; + + default: + sprintf(msg, "we got kill message number %d", reason); + return msg; + } +} + +static int quorum_api_get_quorum(void) +{ + return cluster_is_quorate; +} Index: services/Makefile =================================================================== --- services/Makefile (revision 1663) +++ services/Makefile (working copy) @@ -50,12 +50,12 @@ endif # LCR objects -LCR_SRC = evs.c cfg.c cpg.c confdb.c -LCR_OBJS = evs.o cfg.o cpg.o confdb.o $(AMF_OBJS) +LCR_SRC = evs.c cfg.c cpg.c confdb.c quorum.c +LCR_OBJS = evs.o cfg.o cpg.o confdb.o quorum.o $(AMF_OBJS) override CFLAGS += -fPIC -all: service_evs.lcrso service_cfg.lcrso service_cpg.lcrso service_confdb.lcrso +all: service_evs.lcrso service_cfg.lcrso service_cpg.lcrso service_confdb.lcrso service_quorum.lcrso ifeq (${COROSYNC_COMPAT}, DARWIN) @@ -71,6 +71,9 @@ service_cpg.lcrso: cpg.o $(CC) $(LDFLAGS) -bundle $(LDFLAGS) -bundle_loader ../exec/corosync -bind_at_load cpg.o -o $@ +service_quorum.lcrso: quorum.o + $(CC) $(LDFLAGS) -bundle $(LDFLAGS) -bundle_loader ./aisexec -bind_at_load quorum.o -o $@ + else service_evs.lcrso: evs.o @@ -85,6 +88,9 @@ service_cpg.lcrso: cpg.o $(CC) -shared -Wl,-soname,service_cpg.lcrso cpg.o -o $@ +service_quorum.lcrso: quorum.o + $(CC) -shared -Wl,-soname,service_quorum.lcrso quorum.o -o $@ + endif clean: @@ -104,3 +110,6 @@ cpg.o: cpg.c $(CC) $(CFLAGS) $(CPPFLAGS) -c -o $@ $< + +quorum.o: quorum.c + $(CC) $(CFLAGS) $(CPPFLAGS) -c -o $@ $< Index: services/cmanquorum.h =================================================================== --- services/cmanquorum.h (revision 0) +++ services/cmanquorum.h (revision 0) @@ -0,0 +1,74 @@ +/* + * Copyright (c) 2008 Red Hat, Inc. + * + * All rights reserved. + * + * Author: Christine Caulfield (ccaulfie@redhat.com) + * + * This software licensed under BSD license, the text of which follows: + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * - Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * - Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * - Neither the name of the MontaVista Software, Inc. nor the names of its + * contributors may be used to endorse or promote products derived from this + * software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef QUORUM_H_DEFINED +#define QUORUM_H_DEFINED + +struct quorum_services_api_ver1 { + int (*quorum_api_get_quorum) (void); +}; + +static inline struct quorum_services_api_ver1 * +quorum_services_api_reference ( + struct corosync_api_v1 *coroapi, + unsigned int *handle) +{ + static void *quorum_services_api_p; + struct quorum_services_api_ver1 *return_api; + unsigned int res; + + res = coroapi->plugin_interface_reference ( + handle, + "quorum_services_api", + 0, + &quorum_services_api_p, + 0); + if (res == -1) { + return (NULL); + } + return_api = (struct quorum_services_api_ver1 *)quorum_services_api_p; + return (return_api); +} + +static int inline quorum_services_api_release ( + struct corosync_api_v1 *coroapi, + unsigned int handle) +{ + unsigned int res; + + res = coroapi->plugin_interface_release (handle); + return (res); +} + +#endif /* QUORUM_H_DEFINED */ Index: Makefile.inc =================================================================== --- Makefile.inc (revision 1663) +++ Makefile.inc (working copy) @@ -66,7 +66,7 @@ LDFLAGS += endif ifeq (${COROSYNC_BUILD}, DEBUG) - CFLAGS += -O0 -g -Wall -DDEBUG --time + CFLAGS += -O0 -g -Wall -DDEBUG LDFLAGS += -g ifeq (${COROSYNC_COMPAT}, SOLARIS) CFLAGS += -Werror -DTS_CLASS Index: lib/libquorum.versions =================================================================== --- lib/libquorum.versions (revision 0) +++ lib/libquorum.versions (revision 0) @@ -0,0 +1,32 @@ +# Version and symbol export for libquorum.so + +OPENAIS_QUORUM_1.0 { + global: + quorum_initialize; + quorum_finalize; + quorum_getinfo; + quorum_setexpected; + quorum_setvotes; + quorum_qdisk_register; + quorum_qdisk_unregister; + quorum_qdisk_poll; + quorum_qdisk_getinfo; + quorum_setdirty; + quorum_killnode; + quorum_leaving; + + local: + saHandleCreate; + saHandleDestroy; + saHandleInstanceGet; + saHandleInstancePut; + saRecvRetry; + saSelectRetry; + saSendMsgReceiveReply; + saSendMsgRetry; + saSendReceiveReply; + saSendRetry; + saServiceConnect; + saVersionVerify; + clustTimeNow; +}; Index: lib/cmanquorum.c =================================================================== --- lib/cmanquorum.c (revision 0) +++ lib/cmanquorum.c (revision 0) @@ -0,0 +1,863 @@ +/* + * Copyright (c) 2008 Red Hat, Inc. + * + * All rights reserved. + * + * Author: Christine Caulfield (ccaulfie@redhat.com) + * + * This software licensed under BSD license, the text of which follows: + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * - Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * - Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * - Neither the name of the MontaVista Software, Inc. nor the names of its + * contributors may be used to endorse or promote products derived from this + * software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ +/* + * Provides a quorum API using the corosync executive + */ + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include "corosync/cmanquorum.h" +#include "corosync/ipc_cmanquorum.h" + +struct quorum_inst { + int response_fd; + int dispatch_fd; + int finalize; + void *context; + quorum_callbacks_t callbacks; + pthread_mutex_t response_mutex; + pthread_mutex_t dispatch_mutex; +}; + +static void quorum_instance_destructor (void *instance); + +static struct saHandleDatabase quorum_handle_t_db = { + .handleCount = 0, + .handles = 0, + .mutex = PTHREAD_MUTEX_INITIALIZER, + .handleInstanceDestructor = quorum_instance_destructor +}; + +/* + * Clean up function for a quorum instance (quorum_initialize) handle + */ +static void quorum_instance_destructor (void *instance) +{ + struct quorum_inst *quorum_inst = instance; + + pthread_mutex_destroy (&quorum_inst->response_mutex); +} + +quorum_error_t quorum_initialize ( + quorum_handle_t *handle, + quorum_callbacks_t *callbacks) +{ + SaAisErrorT error; + struct quorum_inst *quorum_inst; + + error = saHandleCreate (&quorum_handle_t_db, sizeof (struct quorum_inst), handle); + if (error != SA_AIS_OK) { + goto error_no_destroy; + } + + error = saHandleInstanceGet (&quorum_handle_t_db, *handle, (void *)&quorum_inst); + if (error != SA_AIS_OK) { + goto error_destroy; + } + + error = saServiceConnect (&quorum_inst->dispatch_fd, + &quorum_inst->response_fd, + QUORUM_SERVICE); + if (error != SA_AIS_OK) { + goto error_put_destroy; + } + + pthread_mutex_init (&quorum_inst->response_mutex, NULL); + pthread_mutex_init (&quorum_inst->dispatch_mutex, NULL); + if (callbacks) + memcpy(&quorum_inst->callbacks, callbacks, sizeof (callbacks)); + else + memset(&quorum_inst->callbacks, 0, sizeof (callbacks)); + + saHandleInstancePut (&quorum_handle_t_db, *handle); + + return (SA_AIS_OK); + +error_put_destroy: + saHandleInstancePut (&quorum_handle_t_db, *handle); +error_destroy: + saHandleDestroy (&quorum_handle_t_db, *handle); +error_no_destroy: + return (error); +} + +quorum_error_t quorum_finalize ( + quorum_handle_t handle) +{ + struct quorum_inst *quorum_inst; + SaAisErrorT error; + + error = saHandleInstanceGet (&quorum_handle_t_db, handle, (void *)&quorum_inst); + if (error != SA_AIS_OK) { + return (error); + } + + pthread_mutex_lock (&quorum_inst->response_mutex); + + /* + * Another thread has already started finalizing + */ + if (quorum_inst->finalize) { + pthread_mutex_unlock (&quorum_inst->response_mutex); + saHandleInstancePut (&quorum_handle_t_db, handle); + return (QUORUM_ERR_BAD_HANDLE); + } + + quorum_inst->finalize = 1; + + pthread_mutex_unlock (&quorum_inst->response_mutex); + + saHandleDestroy (&quorum_handle_t_db, handle); + + /* + * Disconnect from the server + */ + if (quorum_inst->response_fd != -1) { + shutdown(quorum_inst->response_fd, 0); + close(quorum_inst->response_fd); + } + saHandleInstancePut (&quorum_handle_t_db, handle); + + return (QUORUM_OK); +} + + +quorum_error_t quorum_getinfo ( + quorum_handle_t handle, + int nodeid, + struct quorum_info *info) +{ + quorum_error_t error; + struct quorum_inst *quorum_inst; + struct iovec iov[2]; + struct req_lib_quorum_getinfo req_lib_quorum_getinfo; + struct res_lib_quorum_getinfo res_lib_quorum_getinfo; + + error = saHandleInstanceGet (&quorum_handle_t_db, handle, (void *)&quorum_inst); + if (error != SA_AIS_OK) { + return (error); + } + + pthread_mutex_lock (&quorum_inst->response_mutex); + + req_lib_quorum_getinfo.header.size = sizeof (struct req_lib_quorum_getinfo); + req_lib_quorum_getinfo.header.id = MESSAGE_REQ_QUORUM_GETINFO; + req_lib_quorum_getinfo.nodeid = nodeid; + + iov[0].iov_base = (char *)&req_lib_quorum_getinfo; + iov[0].iov_len = sizeof (struct req_lib_quorum_getinfo); + + error = saSendMsgReceiveReply (quorum_inst->response_fd, iov, 1, + &res_lib_quorum_getinfo, sizeof (struct res_lib_quorum_getinfo)); + + pthread_mutex_unlock (&quorum_inst->response_mutex); + + if (error != SA_AIS_OK) { + goto error_exit; + } + + error = res_lib_quorum_getinfo.header.error; + + info->node_id = res_lib_quorum_getinfo.nodeid; + info->node_votes = res_lib_quorum_getinfo.votes; + info->node_expected_votes = res_lib_quorum_getinfo.expected_votes; + info->highest_expected = res_lib_quorum_getinfo.highest_expected; + info->total_votes = res_lib_quorum_getinfo.total_votes; + info->quorum = res_lib_quorum_getinfo.quorum; + info->flags = res_lib_quorum_getinfo.flags; + +error_exit: + saHandleInstancePut (&quorum_handle_t_db, handle); + + return (error); +} + +quorum_error_t quorum_setexpected ( + quorum_handle_t handle, + unsigned int expected_votes) +{ + quorum_error_t error; + struct quorum_inst *quorum_inst; + struct iovec iov[2]; + struct req_lib_quorum_setexpected req_lib_quorum_setexpected; + struct res_lib_quorum_status res_lib_quorum_status; + + error = saHandleInstanceGet (&quorum_handle_t_db, handle, (void *)&quorum_inst); + if (error != SA_AIS_OK) { + return (error); + } + + pthread_mutex_lock (&quorum_inst->response_mutex); + + req_lib_quorum_setexpected.header.size = sizeof (struct req_lib_quorum_setexpected); + req_lib_quorum_setexpected.header.id = MESSAGE_REQ_QUORUM_SETEXPECTED; + req_lib_quorum_setexpected.expected_votes = expected_votes; + + iov[0].iov_base = (char *)&req_lib_quorum_setexpected; + iov[0].iov_len = sizeof (struct req_lib_quorum_setexpected); + + error = saSendMsgReceiveReply (quorum_inst->response_fd, iov, 1, + &res_lib_quorum_status, sizeof (struct res_lib_quorum_status)); + + pthread_mutex_unlock (&quorum_inst->response_mutex); + + if (error != SA_AIS_OK) { + goto error_exit; + } + + error = res_lib_quorum_status.header.error; + +error_exit: + saHandleInstancePut (&quorum_handle_t_db, handle); + + return (error); +} + +quorum_error_t quorum_setvotes ( + quorum_handle_t handle, + int nodeid, + unsigned int votes) +{ + quorum_error_t error; + struct quorum_inst *quorum_inst; + struct iovec iov[2]; + struct req_lib_quorum_setvotes req_lib_quorum_setvotes; + struct res_lib_quorum_status res_lib_quorum_status; + + error = saHandleInstanceGet (&quorum_handle_t_db, handle, (void *)&quorum_inst); + if (error != SA_AIS_OK) { + return (error); + } + + pthread_mutex_lock (&quorum_inst->response_mutex); + + req_lib_quorum_setvotes.header.size = sizeof (struct req_lib_quorum_setvotes); + req_lib_quorum_setvotes.header.id = MESSAGE_REQ_QUORUM_SETVOTES; + req_lib_quorum_setvotes.nodeid = nodeid; + req_lib_quorum_setvotes.votes = votes; + + iov[0].iov_base = (char *)&req_lib_quorum_setvotes; + iov[0].iov_len = sizeof (struct req_lib_quorum_setvotes); + + error = saSendMsgReceiveReply (quorum_inst->response_fd, iov, 1, + &res_lib_quorum_status, sizeof (struct res_lib_quorum_status)); + + pthread_mutex_unlock (&quorum_inst->response_mutex); + + if (error != SA_AIS_OK) { + goto error_exit; + } + + error = res_lib_quorum_status.header.error; + +error_exit: + saHandleInstancePut (&quorum_handle_t_db, handle); + + return (error); +} + +quorum_error_t quorum_qdisk_register ( + quorum_handle_t handle, + char *name, + unsigned int votes) +{ + quorum_error_t error; + struct quorum_inst *quorum_inst; + struct iovec iov[2]; + struct req_lib_quorum_qdisk_register req_lib_quorum_qdisk_register; + struct res_lib_quorum_status res_lib_quorum_status; + + if (strlen(name) > QUORUM_MAX_QDISK_NAME_LEN) + return QUORUM_ERR_INVALID_PARAM; + + error = saHandleInstanceGet (&quorum_handle_t_db, handle, (void *)&quorum_inst); + if (error != SA_AIS_OK) { + return (error); + } + + pthread_mutex_lock (&quorum_inst->response_mutex); + + req_lib_quorum_qdisk_register.header.size = sizeof (struct req_lib_quorum_qdisk_register); + req_lib_quorum_qdisk_register.header.id = MESSAGE_REQ_QUORUM_QDISK_REGISTER; + strcpy(req_lib_quorum_qdisk_register.name, name); + req_lib_quorum_qdisk_register.votes = votes; + + iov[0].iov_base = (char *)&req_lib_quorum_qdisk_register; + iov[0].iov_len = sizeof (struct req_lib_quorum_qdisk_register); + + error = saSendMsgReceiveReply (quorum_inst->response_fd, iov, 1, + &res_lib_quorum_status, sizeof (struct res_lib_quorum_status)); + + pthread_mutex_unlock (&quorum_inst->response_mutex); + + if (error != SA_AIS_OK) { + goto error_exit; + } + + error = res_lib_quorum_status.header.error; + +error_exit: + saHandleInstancePut (&quorum_handle_t_db, handle); + + return (error); +} + +quorum_error_t quorum_qdisk_poll ( + quorum_handle_t handle, + int state) +{ + quorum_error_t error; + struct quorum_inst *quorum_inst; + struct iovec iov[2]; + struct req_lib_quorum_qdisk_poll req_lib_quorum_qdisk_poll; + struct res_lib_quorum_status res_lib_quorum_status; + + error = saHandleInstanceGet (&quorum_handle_t_db, handle, (void *)&quorum_inst); + if (error != SA_AIS_OK) { + return (error); + } + + pthread_mutex_lock (&quorum_inst->response_mutex); + + req_lib_quorum_qdisk_poll.header.size = sizeof (struct req_lib_quorum_qdisk_poll); + req_lib_quorum_qdisk_poll.header.id = MESSAGE_REQ_QUORUM_QDISK_POLL; + req_lib_quorum_qdisk_poll.state = state; + + iov[0].iov_base = (char *)&req_lib_quorum_qdisk_poll; + iov[0].iov_len = sizeof (struct req_lib_quorum_qdisk_poll); + + error = saSendMsgReceiveReply (quorum_inst->response_fd, iov, 1, + &res_lib_quorum_status, sizeof (struct res_lib_quorum_status)); + + pthread_mutex_unlock (&quorum_inst->response_mutex); + + if (error != SA_AIS_OK) { + goto error_exit; + } + + error = res_lib_quorum_status.header.error; + +error_exit: + saHandleInstancePut (&quorum_handle_t_db, handle); + + return (error); +} + +quorum_error_t quorum_qdisk_unregister ( + quorum_handle_t handle) +{ + quorum_error_t error; + struct quorum_inst *quorum_inst; + struct iovec iov[2]; + struct req_lib_quorum_general req_lib_quorum_general; + struct res_lib_quorum_status res_lib_quorum_status; + + error = saHandleInstanceGet (&quorum_handle_t_db, handle, (void *)&quorum_inst); + if (error != SA_AIS_OK) { + return (error); + } + + pthread_mutex_lock (&quorum_inst->response_mutex); + + req_lib_quorum_general.header.size = sizeof (struct req_lib_quorum_general); + req_lib_quorum_general.header.id = MESSAGE_REQ_QUORUM_QDISK_UNREGISTER; + + iov[0].iov_base = (char *)&req_lib_quorum_general; + iov[0].iov_len = sizeof (struct req_lib_quorum_general); + + error = saSendMsgReceiveReply (quorum_inst->response_fd, iov, 1, + &res_lib_quorum_status, sizeof (struct res_lib_quorum_status)); + + pthread_mutex_unlock (&quorum_inst->response_mutex); + + if (error != SA_AIS_OK) { + goto error_exit; + } + + error = res_lib_quorum_status.header.error; + +error_exit: + saHandleInstancePut (&quorum_handle_t_db, handle); + + return (error); +} + + + +quorum_error_t quorum_qdisk_getinfo ( + quorum_handle_t handle, + struct quorum_qdisk_info *qinfo) +{ + quorum_error_t error; + struct quorum_inst *quorum_inst; + struct iovec iov[2]; + struct req_lib_quorum_general req_lib_quorum_general; + struct res_lib_quorum_qdisk_getinfo res_lib_quorum_qdisk_getinfo; + + error = saHandleInstanceGet (&quorum_handle_t_db, handle, (void *)&quorum_inst); + if (error != SA_AIS_OK) { + return (error); + } + + pthread_mutex_lock (&quorum_inst->response_mutex); + + req_lib_quorum_general.header.size = sizeof (struct req_lib_quorum_general); + req_lib_quorum_general.header.id = MESSAGE_REQ_QUORUM_QDISK_GETINFO; + + iov[0].iov_base = (char *)&req_lib_quorum_general; + iov[0].iov_len = sizeof (struct req_lib_quorum_general); + + error = saSendMsgReceiveReply (quorum_inst->response_fd, iov, 1, + &res_lib_quorum_qdisk_getinfo, sizeof (struct res_lib_quorum_qdisk_getinfo)); + + pthread_mutex_unlock (&quorum_inst->response_mutex); + + if (error != SA_AIS_OK) { + goto error_exit; + } + + error = res_lib_quorum_qdisk_getinfo.header.error; + + qinfo->votes = res_lib_quorum_qdisk_getinfo.votes; + qinfo->state = res_lib_quorum_qdisk_getinfo.state; + strcpy(qinfo->name, res_lib_quorum_qdisk_getinfo.name); + + +error_exit: + saHandleInstancePut (&quorum_handle_t_db, handle); + + return (error); +} + +quorum_error_t quorum_setdirty ( + quorum_handle_t handle) +{ + quorum_error_t error; + struct quorum_inst *quorum_inst; + struct iovec iov[2]; + struct req_lib_quorum_general req_lib_quorum_general; + struct res_lib_quorum_status res_lib_quorum_status; + + error = saHandleInstanceGet (&quorum_handle_t_db, handle, (void *)&quorum_inst); + if (error != SA_AIS_OK) { + return (error); + } + + pthread_mutex_lock (&quorum_inst->response_mutex); + + req_lib_quorum_general.header.size = sizeof (struct req_lib_quorum_general); + req_lib_quorum_general.header.id = MESSAGE_REQ_QUORUM_SETDIRTY; + + iov[0].iov_base = (char *)&req_lib_quorum_general; + iov[0].iov_len = sizeof (struct req_lib_quorum_general); + + error = saSendMsgReceiveReply (quorum_inst->response_fd, iov, 1, + &res_lib_quorum_status, sizeof (struct res_lib_quorum_status)); + + pthread_mutex_unlock (&quorum_inst->response_mutex); + + if (error != SA_AIS_OK) { + goto error_exit; + } + + error = res_lib_quorum_status.header.error; + +error_exit: + saHandleInstancePut (&quorum_handle_t_db, handle); + + return (error); +} + +quorum_error_t quorum_leaving ( + quorum_handle_t handle) +{ + quorum_error_t error; + struct quorum_inst *quorum_inst; + struct iovec iov[2]; + struct req_lib_quorum_general req_lib_quorum_general; + struct res_lib_quorum_status res_lib_quorum_status; + + error = saHandleInstanceGet (&quorum_handle_t_db, handle, (void *)&quorum_inst); + if (error != SA_AIS_OK) { + return (error); + } + + pthread_mutex_lock (&quorum_inst->response_mutex); + + req_lib_quorum_general.header.size = sizeof (struct req_lib_quorum_general); + req_lib_quorum_general.header.id = MESSAGE_REQ_QUORUM_LEAVING; + + iov[0].iov_base = (char *)&req_lib_quorum_general; + iov[0].iov_len = sizeof (struct req_lib_quorum_general); + + error = saSendMsgReceiveReply (quorum_inst->response_fd, iov, 1, + &res_lib_quorum_status, sizeof (struct res_lib_quorum_status)); + + pthread_mutex_unlock (&quorum_inst->response_mutex); + + if (error != SA_AIS_OK) { + goto error_exit; + } + + error = res_lib_quorum_status.header.error; + +error_exit: + saHandleInstancePut (&quorum_handle_t_db, handle); + + return (error); +} + +quorum_error_t quorum_killnode ( + quorum_handle_t handle, + int nodeid, + unsigned int reason) +{ + quorum_error_t error; + struct quorum_inst *quorum_inst; + struct iovec iov[2]; + struct req_lib_quorum_killnode req_lib_quorum_killnode; + struct res_lib_quorum_status res_lib_quorum_status; + + error = saHandleInstanceGet (&quorum_handle_t_db, handle, (void *)&quorum_inst); + if (error != SA_AIS_OK) { + return (error); + } + + pthread_mutex_lock (&quorum_inst->response_mutex); + + req_lib_quorum_killnode.header.size = sizeof (struct req_lib_quorum_killnode); + req_lib_quorum_killnode.header.id = MESSAGE_REQ_QUORUM_KILLNODE; + req_lib_quorum_killnode.nodeid = nodeid; + req_lib_quorum_killnode.reason = reason; + + iov[0].iov_base = (char *)&req_lib_quorum_killnode; + iov[0].iov_len = sizeof (struct req_lib_quorum_killnode); + + error = saSendMsgReceiveReply (quorum_inst->response_fd, iov, 1, + &res_lib_quorum_status, sizeof (struct res_lib_quorum_status)); + + pthread_mutex_unlock (&quorum_inst->response_mutex); + + if (error != SA_AIS_OK) { + goto error_exit; + } + + error = res_lib_quorum_status.header.error; + +error_exit: + saHandleInstancePut (&quorum_handle_t_db, handle); + + return (error); +} + +quorum_error_t quorum_trackstart ( + quorum_handle_t handle, + unsigned int flags ) +{ + quorum_error_t error; + struct quorum_inst *quorum_inst; + struct iovec iov[2]; + struct req_lib_quorum_trackstart req_lib_quorum_trackstart; + struct res_lib_quorum_status res_lib_quorum_status; + + error = saHandleInstanceGet (&quorum_handle_t_db, handle, (void *)&quorum_inst); + if (error != SA_AIS_OK) { + return (error); + } + + pthread_mutex_lock (&quorum_inst->response_mutex); + + req_lib_quorum_trackstart.header.size = sizeof (struct req_lib_quorum_trackstart); + req_lib_quorum_trackstart.header.id = MESSAGE_REQ_QUORUM_TRACKSTART; + req_lib_quorum_trackstart.track_flags = flags; + + iov[0].iov_base = (char *)&req_lib_quorum_trackstart; + iov[0].iov_len = sizeof (struct req_lib_quorum_trackstart); + + error = saSendMsgReceiveReply (quorum_inst->response_fd, iov, 1, + &res_lib_quorum_status, sizeof (struct res_lib_quorum_status)); + + pthread_mutex_unlock (&quorum_inst->response_mutex); + + if (error != SA_AIS_OK) { + goto error_exit; + } + + error = res_lib_quorum_status.header.error; + +error_exit: + saHandleInstancePut (&quorum_handle_t_db, handle); + + return (error); +} + +quorum_error_t quorum_trackstop ( + quorum_handle_t handle) +{ + quorum_error_t error; + struct quorum_inst *quorum_inst; + struct iovec iov[2]; + struct req_lib_quorum_general req_lib_quorum_general; + struct res_lib_quorum_status res_lib_quorum_status; + + error = saHandleInstanceGet (&quorum_handle_t_db, handle, (void *)&quorum_inst); + if (error != SA_AIS_OK) { + return (error); + } + + pthread_mutex_lock (&quorum_inst->response_mutex); + + req_lib_quorum_general.header.size = sizeof (struct req_lib_quorum_general); + req_lib_quorum_general.header.id = MESSAGE_REQ_QUORUM_TRACKSTOP; + + iov[0].iov_base = (char *)&req_lib_quorum_general; + iov[0].iov_len = sizeof (struct req_lib_quorum_general); + + error = saSendMsgReceiveReply (quorum_inst->response_fd, iov, 1, + &res_lib_quorum_status, sizeof (struct res_lib_quorum_status)); + + pthread_mutex_unlock (&quorum_inst->response_mutex); + + if (error != SA_AIS_OK) { + goto error_exit; + } + + error = res_lib_quorum_status.header.error; + +error_exit: + saHandleInstancePut (&quorum_handle_t_db, handle); + + return (error); +} + + +quorum_error_t quorum_context_get ( + quorum_handle_t handle, + void **context) +{ + SaAisErrorT error; + struct quorum_inst *quorum_inst; + + error = saHandleInstanceGet (&quorum_handle_t_db, handle, (void *)&quorum_inst); + if (error != SA_AIS_OK) { + return (error); + } + + *context = quorum_inst->context; + + saHandleInstancePut (&quorum_handle_t_db, handle); + + return (SA_AIS_OK); +} + +quorum_error_t quorum_context_set ( + quorum_handle_t handle, + void *context) +{ + SaAisErrorT error; + struct quorum_inst *quorum_inst; + + error = saHandleInstanceGet (&quorum_handle_t_db, handle, (void *)&quorum_inst); + if (error != SA_AIS_OK) { + return (error); + } + + quorum_inst->context = context; + + saHandleInstancePut (&quorum_handle_t_db, handle); + + return (SA_AIS_OK); +} + + +struct res_overlay { + mar_res_header_t header __attribute__((aligned(8))); + char data[512000]; +}; + +quorum_error_t quorum_dispatch ( + quorum_handle_t handle, + quorum_dispatch_t dispatch_types) +{ + struct pollfd ufds; + int timeout = -1; + SaAisErrorT error; + int cont = 1; /* always continue do loop except when set to 0 */ + int dispatch_avail; + struct quorum_inst *quorum_inst; + quorum_callbacks_t callbacks; + struct res_overlay dispatch_data; + struct res_lib_quorum_notification *res_lib_quorum_notification; + int items_to_copy; + unsigned int i; + + if (dispatch_types != SA_DISPATCH_ONE && + dispatch_types != SA_DISPATCH_ALL && + dispatch_types != SA_DISPATCH_BLOCKING) { + + return (SA_AIS_ERR_INVALID_PARAM); + } + + error = saHandleInstanceGet (&quorum_handle_t_db, handle, + (void *)&quorum_inst); + if (error != SA_AIS_OK) { + return (error); + } + + /* + * Timeout instantly for SA_DISPATCH_ONE or SA_DISPATCH_ALL and + * wait indefinately for SA_DISPATCH_BLOCKING + */ + if (dispatch_types == SA_DISPATCH_ALL) { + timeout = 0; + } + + do { + ufds.fd = quorum_inst->dispatch_fd; + ufds.events = POLLIN; + ufds.revents = 0; + + pthread_mutex_lock (&quorum_inst->dispatch_mutex); + + error = saPollRetry (&ufds, 1, timeout); + if (error != SA_AIS_OK) { + goto error_unlock; + } + + /* + * Handle has been finalized in another thread + */ + if (quorum_inst->finalize == 1) { + error = SA_AIS_OK; + goto error_unlock; + } + + if ((ufds.revents & (POLLERR|POLLHUP|POLLNVAL)) != 0) { + error = SA_AIS_ERR_BAD_HANDLE; + goto error_unlock; + } + + dispatch_avail = ufds.revents & POLLIN; + if (dispatch_avail == 0 && dispatch_types == SA_DISPATCH_ALL) { + pthread_mutex_unlock (&quorum_inst->dispatch_mutex); + break; /* exit do while cont is 1 loop */ + } else + if (dispatch_avail == 0) { + pthread_mutex_unlock (&quorum_inst->dispatch_mutex); + continue; /* next poll */ + } + + if (ufds.revents & POLLIN) { + error = saRecvRetry (quorum_inst->dispatch_fd, &dispatch_data.header, + sizeof (mar_res_header_t)); + if (error != SA_AIS_OK) { + goto error_unlock; + } + if (dispatch_data.header.size > sizeof (mar_res_header_t)) { + error = saRecvRetry (quorum_inst->dispatch_fd, &dispatch_data.data, + dispatch_data.header.size - sizeof (mar_res_header_t)); + if (error != SA_AIS_OK) { + goto error_unlock; + } + } + } else { + pthread_mutex_unlock (&quorum_inst->dispatch_mutex); + continue; + } + + /* + * Make copy of callbacks, message data, unlock instance, and call callback + * A risk of this dispatch method is that the callback routines may + * operate at the same time that quorum_finalize has been called in another thread. + */ + memcpy (&callbacks, &quorum_inst->callbacks, sizeof (quorum_callbacks_t)); + pthread_mutex_unlock (&quorum_inst->dispatch_mutex); + + /* + * Dispatch incoming message + */ + switch (dispatch_data.header.id) { + + case MESSAGE_RES_QUORUM_NOTIFICATION: + if (callbacks.quorum_notify_fn == NULL) { + continue; + } + res_lib_quorum_notification = (struct res_lib_quorum_notification *)&dispatch_data; + + callbacks.quorum_notify_fn ( handle, + res_lib_quorum_notification->quorate, + res_lib_quorum_notification->node_list_entries, + (quorum_node_t *)res_lib_quorum_notification->node_list ); + ; + break; + + default: + error = SA_AIS_ERR_LIBRARY; + goto error_put; + break; + } + + /* + * Determine if more messages should be processed + * */ + switch (dispatch_types) { + case SA_DISPATCH_ONE: + cont = 0; + break; + case SA_DISPATCH_ALL: + break; + case SA_DISPATCH_BLOCKING: + break; + } + } while (cont); + + goto error_put; + +error_unlock: + pthread_mutex_unlock (&quorum_inst->dispatch_mutex); + +error_put: + saHandleInstancePut (&quorum_handle_t_db, handle); + return (error); +} Index: lib/Makefile =================================================================== --- lib/Makefile (revision 1663) +++ lib/Makefile (working copy) @@ -41,6 +41,7 @@ libconfdb.a libconfdb.so.2.0.0 \ libevs.a libevs.so.2.0.0 \ libcfg.a libcfg.so.2.0.0 \ + libquorum.a libquorum.so.2.0.0 \ libcoroutil.a libcoroutil.so.2.0.0 libcoroutil.a: util.o @@ -67,6 +68,9 @@ libcpg.so.2.0.0: util.o cpg.o $(CC) $(DARWIN_OPTS) util.o cpg.o -o $@ +libquorum.so.2.0.0: util.o quorum.o + $(CC) $(DARWIN_OPTS) util.o quorum.o -o $@ + else libcoroutil.so.2.0.0: util.o @@ -84,6 +88,9 @@ libcfg.so.2.0.0: util.o cfg.o $(CC) -shared -Wl,-soname,libcfg.so.2,-version-script=$(srcdir)$(subdir)libcfg.versions util.o cfg.o -o $@ +libquorum.so.2.0.0: util.o quorum.o + $(CC) -shared -Wl,-soname,libquorum.so.2,-version-script=$(srcdir)$(subdir)libquorum.versions util.o quorum.o -o $@ + endif libevs.a: util.o evs.o @@ -98,10 +105,14 @@ libcfg.a: util.o cfg.o $(AR) -rc libcfg.a util.o cfg.o +libquorum.a: util.o quorum.o + $(AR) -rc libquorum.a util.o quorum.o + clean: rm -f *.o libcfg.so* libcoroutil.so* libcoroutil.a \ libevs.so* libevs.a libcpg.so* libcpg.a libcfg.a libconfdb.so* \ - libconfdb.a libconfdb.a \ *.da *.bb *.bbg + libquorum.so.* libquorum.a \ + libconfdb.a libconfdb.a *.da *.bb *.bbg # -fPIC rules required for all libraries %.o: %.c