From b4a2b86dc975dade9005443b27e3805f5fffc694 Mon Sep 17 00:00:00 2001 From: Amitay Isaacs Date: Thu, 11 Feb 2016 14:32:34 +1100 Subject: [PATCH 01/29] ctdb-recovery: Create recovery databases in state dir This matches the behaviour during serial database recovery. Signed-off-by: Amitay Isaacs Reviewed-by: Martin Schwenke Autobuild-User(master): Martin Schwenke Autobuild-Date(master): Thu Feb 11 08:01:14 CET 2016 on sn-devel-144 (cherry picked from commit 19a411f839c5c34fec2aa5e6cb095346be56d94e) --- ctdb/server/ctdb_recoverd.c | 2 ++ ctdb/server/ctdb_recovery_helper.c | 9 ++++++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/ctdb/server/ctdb_recoverd.c b/ctdb/server/ctdb_recoverd.c index c89649a..e42433d 100644 --- a/ctdb/server/ctdb_recoverd.c +++ b/ctdb/server/ctdb_recoverd.c @@ -1781,6 +1781,8 @@ static int db_recovery_parallel(struct ctdb_recoverd *rec, TALLOC_CTX *mem_ctx) goto fail; } + setenv("CTDB_DBDIR_STATE", rec->ctdb->db_directory_state, 1); + if (!ctdb_vfork_with_logging(state, rec->ctdb, "recovery", prog, nargs, args, NULL, NULL, &state->pid)) { DEBUG(DEBUG_ERR, diff --git a/ctdb/server/ctdb_recovery_helper.c b/ctdb/server/ctdb_recovery_helper.c index 6d6a835..1ddddfc 100644 --- a/ctdb/server/ctdb_recovery_helper.c +++ b/ctdb/server/ctdb_recovery_helper.c @@ -80,6 +80,7 @@ static struct recdb_context *recdb_create(TALLOC_CTX *mem_ctx, uint32_t db_id, const char *db_path, uint32_t hash_size, bool persistent) { + static char *db_dir_state = NULL; struct recdb_context *recdb; unsigned int tdb_flags; @@ -88,10 +89,16 @@ static struct recdb_context *recdb_create(TALLOC_CTX *mem_ctx, uint32_t db_id, return NULL; } + if (db_dir_state == NULL) { + db_dir_state = getenv("CTDB_DBDIR_STATE"); + } + recdb->db_name = db_name; recdb->db_id = db_id; recdb->db_path = talloc_asprintf(recdb, "%s/recdb.%s", - dirname(discard_const(db_path)), + db_dir_state != NULL ? + db_dir_state : + dirname(discard_const(db_path)), db_name); if (recdb->db_path == NULL) { talloc_free(recdb); -- 2.5.5 From 45f62ac8c8e50c3ff37359919bc3cf249aada1ce Mon Sep 17 00:00:00 2001 From: Amitay Isaacs Date: Wed, 10 Feb 2016 11:39:37 +1100 Subject: [PATCH 02/29] ctdb-recovery: Add a log message when marshalling recovery database fails Signed-off-by: Amitay Isaacs Reviewed-by: Martin Schwenke (cherry picked from commit 157e19b9848c4745e2a9a2169e25698cfd4dcaa6) --- ctdb/server/ctdb_recovery_helper.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ctdb/server/ctdb_recovery_helper.c b/ctdb/server/ctdb_recovery_helper.c index 1ddddfc..f0fa12d 100644 --- a/ctdb/server/ctdb_recovery_helper.c +++ b/ctdb/server/ctdb_recovery_helper.c @@ -270,6 +270,8 @@ static struct ctdb_rec_buffer *recdb_records(struct recdb_context *recdb, ret = tdb_traverse_read(recdb->db->tdb, recdb_traverse, &state); if (ret == -1 || state.failed) { + LOG("Failed to marshall recovery records for %s\n", + recdb->db_name); TALLOC_FREE(state.recbuf); return NULL; } -- 2.5.5 From edf6473236d4fbe545c74f417b33689132d756e8 Mon Sep 17 00:00:00 2001 From: Amitay Isaacs Date: Wed, 18 Nov 2015 15:11:37 +1100 Subject: [PATCH 03/29] ctdb-client: Add async version of set/remove message handler functions Signed-off-by: Amitay Isaacs Reviewed-by: Martin Schwenke (cherry picked from commit d23c5a6c2f365b249233f9a1c6d42d220b2b7c67) --- ctdb/client/client.h | 18 +++++ ctdb/client/client_message.c | 179 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 197 insertions(+) diff --git a/ctdb/client/client.h b/ctdb/client/client.h index bce0c6b..48b69b6 100644 --- a/ctdb/client/client.h +++ b/ctdb/client/client.h @@ -76,6 +76,24 @@ int ctdb_client_message(TALLOC_CTX *mem_ctx, struct tevent_context *ev, struct ctdb_client_context *client, uint32_t destnode, struct ctdb_req_message *message); +struct tevent_req *ctdb_client_set_message_handler_send( + TALLOC_CTX *mem_ctx, + struct tevent_context *ev, + struct ctdb_client_context *client, + uint64_t srvid, + srvid_handler_fn handler, + void *private_data); +bool ctdb_client_set_message_handler_recv(struct tevent_req *req, int *perr); + +struct tevent_req *ctdb_client_remove_message_handler_send( + TALLOC_CTX *mem_ctx, + struct tevent_context *ev, + struct ctdb_client_context *client, + uint64_t srvid, + void *private_data); +bool ctdb_client_remove_message_handler_recv(struct tevent_req *req, + int *perr); + int ctdb_client_set_message_handler(TALLOC_CTX *mem_ctx, struct tevent_context *ev, struct ctdb_client_context *client, diff --git a/ctdb/client/client_message.c b/ctdb/client/client_message.c index bfa9ba2..c316f42 100644 --- a/ctdb/client/client_message.c +++ b/ctdb/client/client_message.c @@ -192,6 +192,185 @@ int ctdb_client_message(TALLOC_CTX *mem_ctx, struct tevent_context *ev, return 0; } +struct ctdb_client_set_message_handler_state { + struct ctdb_client_context *client; + uint64_t srvid; + srvid_handler_fn handler; + void *private_data; +}; + +static void ctdb_client_set_message_handler_done(struct tevent_req *subreq); + +struct tevent_req *ctdb_client_set_message_handler_send( + TALLOC_CTX *mem_ctx, + struct tevent_context *ev, + struct ctdb_client_context *client, + uint64_t srvid, + srvid_handler_fn handler, + void *private_data) +{ + struct tevent_req *req, *subreq; + struct ctdb_client_set_message_handler_state *state; + struct ctdb_req_control request; + + req = tevent_req_create(mem_ctx, &state, + struct ctdb_client_set_message_handler_state); + if (req == NULL) { + return NULL; + } + + state->client = client; + state->srvid = srvid; + state->handler = handler; + state->private_data = private_data; + + ctdb_req_control_register_srvid(&request, srvid); + subreq = ctdb_client_control_send(state, ev, client, client->pnn, + tevent_timeval_zero(), &request); + if (tevent_req_nomem(subreq, req)) { + return tevent_req_post(req, ev); + } + tevent_req_set_callback(subreq, ctdb_client_set_message_handler_done, + req); + + return req; +} + +static void ctdb_client_set_message_handler_done(struct tevent_req *subreq) +{ + struct tevent_req *req = tevent_req_callback_data( + subreq, struct tevent_req); + struct ctdb_client_set_message_handler_state *state = tevent_req_data( + req, struct ctdb_client_set_message_handler_state); + struct ctdb_reply_control *reply; + bool status; + int ret; + + status = ctdb_client_control_recv(subreq, &ret, state, &reply); + TALLOC_FREE(subreq); + if (! status) { + tevent_req_error(req, ret); + return; + } + + ret = ctdb_reply_control_register_srvid(reply); + talloc_free(reply); + if (ret != 0) { + tevent_req_error(req, ret); + return; + } + + ret = srvid_register(state->client->srv, state->client, state->srvid, + state->handler, state->private_data); + if (ret != 0) { + tevent_req_error(req, ret); + return; + } + + tevent_req_done(req); +} + +bool ctdb_client_set_message_handler_recv(struct tevent_req *req, int *perr) +{ + int err; + + if (tevent_req_is_unix_error(req, &err)) { + if (perr != NULL) { + *perr = err; + } + return false; + } + return true; +} + +struct ctdb_client_remove_message_handler_state { + struct ctdb_client_context *client; + uint64_t srvid; + void *private_data; +}; + +static void ctdb_client_remove_message_handler_done(struct tevent_req *subreq); + +struct tevent_req *ctdb_client_remove_message_handler_send( + TALLOC_CTX *mem_ctx, + struct tevent_context *ev, + struct ctdb_client_context *client, + uint64_t srvid, + void *private_data) +{ + struct tevent_req *req, *subreq; + struct ctdb_client_remove_message_handler_state *state; + struct ctdb_req_control request; + + req = tevent_req_create(mem_ctx, &state, + struct ctdb_client_remove_message_handler_state); + if (req == NULL) { + return NULL; + } + + state->client = client; + state->srvid = srvid; + state->private_data = private_data; + + ctdb_req_control_deregister_srvid(&request, srvid); + subreq = ctdb_client_control_send(state, ev, client, client->pnn, + tevent_timeval_zero(), &request); + if (tevent_req_nomem(subreq, req)) { + return tevent_req_post(req, ev); + } + tevent_req_set_callback(subreq, + ctdb_client_remove_message_handler_done, req); + + return req; +} + +static void ctdb_client_remove_message_handler_done(struct tevent_req *subreq) +{ + struct tevent_req *req = tevent_req_callback_data( + subreq, struct tevent_req); + struct ctdb_client_remove_message_handler_state *state = tevent_req_data( + req, struct ctdb_client_remove_message_handler_state); + struct ctdb_reply_control *reply; + bool status; + int ret; + + status = ctdb_client_control_recv(subreq, &ret, state, &reply); + TALLOC_FREE(subreq); + if (! status) { + tevent_req_error(req, ret); + return; + } + + ret = ctdb_reply_control_deregister_srvid(reply); + talloc_free(reply); + if (ret != 0) { + tevent_req_error(req, ret); + return; + } + + ret = srvid_deregister(state->client->srv, state->srvid, + state->private_data); + if (ret != 0) { + tevent_req_error(req, ret); + return; + } + + tevent_req_done(req); +} + +bool ctdb_client_remove_message_handler_recv(struct tevent_req *req, int *perr) +{ + int err; + + if (tevent_req_is_unix_error(req, &err)) { + if (perr != NULL) { + *perr = err; + } + return false; + } + return true; +} + int ctdb_client_set_message_handler(TALLOC_CTX *mem_ctx, struct tevent_context *ev, struct ctdb_client_context *client, -- 2.5.5 From b835d3b6b05a8ea5fe042a5dfe337435b6fca9b7 Mon Sep 17 00:00:00 2001 From: Amitay Isaacs Date: Mon, 7 Mar 2016 17:05:11 +1100 Subject: [PATCH 04/29] ctdb-doc: Sort the tunable variables in alphabetical order Signed-off-by: Amitay Isaacs Reviewed-by: Martin Schwenke (cherry picked from commit 4bf6cab4a1e1c11f79fde40fc8b9ae29b0f00e35) --- ctdb/doc/ctdb-tunables.7.xml | 658 ++++++++++++++++++++++--------------------- 1 file changed, 332 insertions(+), 326 deletions(-) diff --git a/ctdb/doc/ctdb-tunables.7.xml b/ctdb/doc/ctdb-tunables.7.xml index 6c164f3..a6ef435 100644 --- a/ctdb/doc/ctdb-tunables.7.xml +++ b/ctdb/doc/ctdb-tunables.7.xml @@ -29,39 +29,17 @@ getvar commands for more details. - - MaxRedirectCount - Default: 3 - - If we are not the DMASTER and need to fetch a record across the network - we first send the request to the LMASTER after which the record - is passed onto the current DMASTER. If the DMASTER changes before - the request has reached that node, the request will be passed onto the - "next" DMASTER. For very hot records that migrate rapidly across the - cluster this can cause a request to "chase" the record for many hops - before it catches up with the record. - - this is how many hops we allow trying to chase the DMASTER before we - switch back to the LMASTER again to ask for new directions. - - - When chasing a record, this is how many hops we will chase the record - for before going back to the LMASTER to ask for new guidance. - - + + The tunable variables are listed alphabetically. + - SeqnumInterval - Default: 1000 - - Some databases have seqnum tracking enabled, so that samba will be able - to detect asynchronously when there has been updates to the database. - Everytime a database is updated its sequence number is increased. - + AllowClientDBAttach + Default: 1 - This tunable is used to specify in 'ms' how frequently ctdb will - send out updates to remote nodes to inform them that the sequence - number is increased. + When set to 0, clients are not allowed to attach to any databases. + This can be used to temporarily block any new processes from + attaching to and accessing the databases. @@ -69,98 +47,130 @@ ControlTimeout Default: 60 - This is the default - setting for timeout for when sending a control message to either the - local or a remote ctdb daemon. + This is the default setting for timeout for when sending a + control message to either the local or a remote ctdb daemon. - TraverseTimeout - Default: 20 + DatabaseHashSize + Default: 100001 - This setting controls how long we allow a traverse process to run. - After this timeout triggers, the main ctdb daemon will abort the - traverse if it has not yet finished. + Number of the hash chains for the local store of the tdbs that + ctdb manages. - KeepaliveInterval + DatabaseMaxDead Default: 5 - How often in seconds should the nodes send keepalives to eachother. + How many dead records per hashchain in the TDB database do we + allow before the freelist needs to be processed. - KeepaliveLimit - Default: 5 + DBRecordCountWarn + Default: 100000 - After how many keepalive intervals without any traffic should a node - wait until marking the peer as DISCONNECTED. + When set to non-zero, ctdb will log a warning when we try + to recover a database with more than this many records. This + will produce a warning if a database grows uncontrollably with + orphaned records. + + + + DBRecordSizeWarn + Default: 10000000 - If a node has hung, it can thus take KeepaliveInterval*(KeepaliveLimit+1) - seconds before we determine that the node is DISCONNECTED and that we - require a recovery. This limitshould not be set too high since we want - a hung node to be detectec, and expunged from the cluster well before - common CIFS timeouts (45-90 seconds) kick in. + When set to non-zero, ctdb will log a warning when we try to + recover a database where a single record is bigger than this. This + will produce a warning if a database record grows uncontrollably + with orphaned sub-records. - RecoverTimeout - Default: 20 + DBSizeWarn + Default: 1000000000 - This is the default setting for timeouts for controls when sent from the - recovery daemon. We allow longer control timeouts from the recovery daemon - than from normal use since the recovery dameon often use controls that - can take a lot longer than normal controls. + When set to non-zero, ctdb will log a warning when we try to + recover a database bigger than this. This will produce a warning + if a database grows uncontrollably. - RecoverInterval - Default: 1 + DeferredAttachTO + Default: 120 + + When databases are frozen we do not allow clients to attach to + the databases. Instead of returning an error immediately to the + application the attach request from the client is deferred until + the database becomes available again at which stage we respond + to the client. + - How frequently in seconds should the recovery daemon perform the - consistency checks that determine if we need to perform a recovery or not. + This timeout controls how long we will defer the request from the + client before timing it out and returning an error to the client. - ElectionTimeout - Default: 3 + DeterministicIPs + Default: 0 - When electing a new recovery master, this is how many seconds we allow - the election to take before we either deem the election finished - or we fail the election and start a new one. + When enabled, this tunable makes ctdb try to keep public IP + addresses locked to specific nodes as far as possible. This makes + it easier for debugging since you can know that as long as all + nodes are healthy public IP X will always be hosted by node Y. + + + The cost of using deterministic IP address assignment is that it + disables part of the logic where ctdb tries to reduce the number + of public IP assignment changes in the cluster. This tunable may + increase the number of IP failover/failbacks that are performed + on the cluster by a small margin. - TakeoverTimeout - Default: 9 + DisableIPFailover + Default: 0 + + When enabled, ctdb will not perform failover or failback. Even + if a node fails while holding public IPs, ctdb will not recover + the IPs or assign them to another node. + - This is how many seconds we allow controls to take for IP failover events. + When you enable this tunable, CTDB will no longer attempt + to recover the cluster by failing IP addresses over to other + nodes. This leads to a service outage until the administrator + has manually performed failover to replacement nodes using the + 'ctdb moveip' command. - MonitorInterval - Default: 15 + ElectionTimeout + Default: 3 - How often should ctdb run the event scripts to check for a nodes health. + When electing a new recovery master, this is how many seconds + we allow the election to take before we either deem the election + finished or we fail the election and start a new one. - TickleUpdateInterval - Default: 20 + EnableBans + Default: 1 - How often will ctdb record and store the "tickle" information used to - kickstart stalled tcp connections after a recovery. + When set to 0, this disables BANNING completely in the cluster + and thus nodes can not get banned, even it they break. Don't + set to 0 unless you know what you are doing. You should set + this to the same value on all nodes to avoid unexpected behaviour. @@ -172,7 +182,6 @@ out. This is the total time for all enabled scripts that are run for an event, not just a single event script. - Note that timeouts are ignored for some events ("takeip", "releaseip", "startrecovery", "recovered") and converted to @@ -182,162 +191,157 @@ - MonitorTimeoutCount - Default: 20 + FetchCollapse + Default: 1 - How many monitor events in a row need to timeout before a node - is flagged as UNHEALTHY. This setting is useful if scripts - can not be written so that they do not hang for benign - reasons. + When many clients across many nodes try to access the same record + at the same time this can lead to a fetch storm where the record + becomes very active and bounces between nodes very fast. This + leads to high CPU utilization of the ctdbd daemon, trying to + bounce that record around very fast, and poor performance. - - - - RecoveryGracePeriod - Default: 120 - During recoveries, if a node has not caused recovery failures during the - last grace period, any records of transgressions that the node has caused - recovery failures will be forgiven. This resets the ban-counter back to - zero for that node. + This parameter is used to activate a fetch-collapse. A + fetch-collapse is when we track which records we have requests in + flight so that we only keep one request in flight from a certain + node, even if multiple smbd processes are attemtping to fetch + the record at the same time. This can improve performance and + reduce CPU utilization for certain workloads. - - - - RecoveryBanPeriod - Default: 300 - If a node becomes banned causing repetitive recovery failures. The node will - eventually become banned from the cluster. - This controls how long the culprit node will be banned from the cluster - before it is allowed to try to join the cluster again. - Don't set to small. A node gets banned for a reason and it is usually due - to real problems with the node. + This timeout controls if we should collapse multiple fetch + operations of the same record into a single request and defer + all duplicates or not. - DatabaseHashSize - Default: 100001 + HopcountMakeSticky + Default: 50 - Size of the hash chains for the local store of the tdbs that ctdb manages. + If the database is set to 'STICKY' mode, using the 'ctdb + setdbsticky' command, any record that is seen as very hot and + migrating so fast that hopcount surpasses 50 is set to become a + STICKY record for StickyDuration seconds. This means that after + each migration the record will be kept on the node and prevented + from being migrated off the node. + + + This setting allows one to try to identify such records and + stop them from migrating across the cluster so fast. This will + improve performance for certain workloads, such as locking.tdb + if many clients are opening/closing the same file concurrently. - DatabaseMaxDead + KeepaliveInterval Default: 5 - How many dead records per hashchain in the TDB database do we allow before - the freelist needs to be processed. + How often in seconds should the nodes send keep-alive packets to + each other. - RerecoveryTimeout - Default: 10 + KeepaliveLimit + Default: 5 - Once a recovery has completed, no additional recoveries are permitted - until this timeout has expired. + After how many keepalive intervals without any traffic should + a node wait until marking the peer as DISCONNECTED. + + + If a node has hung, it can thus take + KeepaliveInterval*(KeepaliveLimit+1) seconds before we determine + that the node is DISCONNECTED and that we require a recovery. This + limitshould not be set too high since we want a hung node to be + detectec, and expunged from the cluster well before common CIFS + timeouts (45-90 seconds) kick in. - EnableBans + LCP2PublicIPs Default: 1 - When set to 0, this disables BANNING completely in the cluster and thus - nodes can not get banned, even it they break. Don't set to 0 unless you - know what you are doing. You should set this to the same value on - all nodes to avoid unexpected behaviour. + When enabled this switches ctdb to use the LCP2 ip allocation + algorithm. - DeterministicIPs + LogLatencyMs Default: 0 - When enabled, this tunable makes ctdb try to keep public IP addresses - locked to specific nodes as far as possible. This makes it easier for - debugging since you can know that as long as all nodes are healthy - public IP X will always be hosted by node Y. - - - The cost of using deterministic IP address assignment is that it - disables part of the logic where ctdb tries to reduce the number of - public IP assignment changes in the cluster. This tunable may increase - the number of IP failover/failbacks that are performed on the cluster - by a small margin. + When set to non-zero, this will make the main daemon log any + operation that took longer than this value, in 'ms', to complete. + These include "how long time a lockwait child process needed", + "how long time to write to a persistent database" but also "how + long did it take to get a response to a CALL from a remote node". - + - LCP2PublicIPs - Default: 1 + MaxRedirectCount + Default: 3 - When enabled this switches ctdb to use the LCP2 ip allocation - algorithm. + If we are not the DMASTER and need to fetch a record across the + network we first send the request to the LMASTER after which the + record is passed onto the current DMASTER. If the DMASTER changes + before the request has reached that node, the request will be + passed onto the "next" DMASTER. For very hot records that migrate + rapidly across the cluster this can cause a request to "chase" + the record for many hops before it catches up with the record. + + + When chasing a record, this is how many hops we will chase + the record for before going back to the LMASTER to ask for + new guidance. - ReclockPingPeriod - Default: x + MonitorInterval + Default: 15 - Obsolete + How often should ctdb run the event scripts in seconds to check + for a node's health. - NoIPFailback - Default: 0 - - When set to 1, ctdb will not perform failback of IP addresses when a node - becomes healthy. Ctdb WILL perform failover of public IP addresses when a - node becomes UNHEALTHY, but when the node becomes HEALTHY again, ctdb - will not fail the addresses back. - - - Use with caution! Normally when a node becomes available to the cluster - ctdb will try to reassign public IP addresses onto the new node as a way - to distribute the workload evenly across the clusternode. Ctdb tries to - make sure that all running nodes have approximately the same number of - public addresses it hosts. - + MonitorTimeoutCount + Default: 20 - When you enable this tunable, CTDB will no longer attempt to rebalance - the cluster by failing IP addresses back to the new nodes. An unbalanced - cluster will therefore remain unbalanced until there is manual - intervention from the administrator. When this parameter is set, you can - manually fail public IP addresses over to the new node(s) using the - 'ctdb moveip' command. + How many monitor events in a row need to timeout before a node + is flagged as UNHEALTHY. This setting is useful if scripts can + not be written so that they do not hang for benign reasons. - DisableIPFailover + NoIPFailback Default: 0 - When enabled, ctdb will not perform failover or failback. Even if a - node fails while holding public IPs, ctdb will not recover the IPs or - assign them to another node. + When set to 1, ctdb will not perform failback of IP addresses + when a node becomes healthy. Ctdb WILL perform failover of public + IP addresses when a node becomes UNHEALTHY, but when the node + becomes HEALTHY again, ctdb will not fail the addresses back. - When you enable this tunable, CTDB will no longer attempt to recover - the cluster by failing IP addresses over to other nodes. This leads to - a service outage until the administrator has manually performed failover - to replacement nodes using the 'ctdb moveip' command. + Use with caution! Normally when a node becomes available to the + cluster ctdb will try to reassign public IP addresses onto the + new node as a way to distribute the workload evenly across the + clusternode. Ctdb tries to make sure that all running nodes have + approximately the same number of public addresses it hosts. - - - - NoIPTakeover - Default: 0 - When set to 1, ctdb will not allow IP addresses to be failed over - onto this node. Any IP addresses that the node currently hosts - will remain on the node but no new IP addresses can be failed over - to the node. + When you enable this tunable, CTDB will no longer attempt to + rebalance the cluster by failing IP addresses back to the new + nodes. An unbalanced cluster will therefore remain unbalanced + until there is manual intervention from the administrator. When + this parameter is set, you can manually fail public IP addresses + over to the new node(s) using the 'ctdb moveip' command. @@ -347,7 +351,7 @@ If no nodes are healthy then by default ctdb will happily host public IPs on disabled (unhealthy or administratively disabled) - nodes. This can cause problems, for example if the underlying + nodes. This can cause problems, for example if the underlying cluster filesystem is not mounted. When set to 1 on a node and that node is disabled it, any IPs hosted by this node will be released and the node will not takeover any IPs until it is no @@ -356,85 +360,100 @@ - DBRecordCountWarn - Default: 100000 + NoIPTakeover + Default: 0 - When set to non-zero, ctdb will log a warning when we try to recover a - database with more than this many records. This will produce a warning - if a database grows uncontrollably with orphaned records. + When set to 1, ctdb will not allow IP addresses to be failed + over onto this node. Any IP addresses that the node currently + hosts will remain on the node but no new IP addresses can be + failed over to the node. - DBRecordSizeWarn - Default: 10000000 + RecdFailCount + Default: 10 - When set to non-zero, ctdb will log a warning when we try to recover a - database where a single record is bigger than this. This will produce - a warning if a database record grows uncontrollably with orphaned - sub-records. + If the recovery daemon has failed to ping the main dameon for + this many consecutive intervals, the main daemon will consider + the recovery daemon as hung and will try to restart it to recover. - DBSizeWarn - Default: 1000000000 + RecdPingTimeout + Default: 60 - When set to non-zero, ctdb will log a warning when we try to recover a - database bigger than this. This will produce - a warning if a database grows uncontrollably. + If the main dameon has not heard a "ping" from the recovery dameon + for this many seconds, the main dameon will log a message that + the recovery daemon is potentially hung. - VerboseMemoryNames - Default: 0 + RecLockLatencyMs + Default: 1000 - This feature consumes additional memory. when used the talloc library - will create more verbose names for all talloc allocated objects. + When using a reclock file for split brain prevention, if set + to non-zero this tunable will make the recovery dameon log a + message if the fcntl() call to lock/testlock the recovery file + takes longer than this number of ms. - RecdPingTimeout - Default: 60 + RecoverInterval + Default: 1 - If the main dameon has not heard a "ping" from the recovery dameon for - this many seconds, the main dameon will log a message that the recovery - daemon is potentially hung. + How frequently in seconds should the recovery daemon perform + the consistency checks that determine if we need to perform a + recovery or not. - RecdFailCount - Default: 10 + RecoverPDBBySeqNum + Default: 1 + + When set to zero, database recovery for persistent databases is + record-by-record and recovery process simply collects the most + recent version of every individual record. + + + When set to non-zero, persistent databases will instead be + recovered as a whole db and not by individual records. The + node that contains the highest value stored in the record + "__db_sequence_number__" is selected and the copy of that nodes + database is used as the recovered database. + - If the recovery daemon has failed to ping the main dameon for this many - consecutive intervals, the main daemon will consider the recovery daemon - as hung and will try to restart it to recover. + By default, recovery of persistent databses is done using + __db_sequence_number__ record. - LogLatencyMs - Default: 0 + RecoverTimeout + Default: 20 - When set to non-zero, this will make the main daemon log any operation that - took longer than this value, in 'ms', to complete. - These include "how long time a lockwait child process needed", - "how long time to write to a persistent database" but also - "how long did it take to get a response to a CALL from a remote node". + This is the default setting for timeouts for controls when sent + from the recovery daemon. We allow longer control timeouts from + the recovery daemon than from normal use since the recovery + dameon often use controls that can take a lot longer than normal + controls. - RecLockLatencyMs - Default: 1000 + RecoveryBanPeriod + Default: 300 - When using a reclock file for split brain prevention, if set to non-zero - this tunable will make the recovery dameon log a message if the fcntl() - call to lock/testlock the recovery file takes longer than this number of - ms. + If a node becomes banned causing repetitive recovery failures. The + node will eventually become banned from the cluster. This + controls how long the culprit node will be banned from the + cluster before it is allowed to try to join the cluster again. + Don't set to small. A node gets banned for a reason and it is + usually due to real problems with the node. @@ -448,21 +467,13 @@ - VacuumInterval - Default: 10 - - Periodic interval in seconds when vacuuming is triggered for - volatile databases. - - - - - VacuumMaxRunTime + RecoveryGracePeriod Default: 120 - The maximum time in seconds for which the vacuuming process is - allowed to run. If vacuuming process takes longer than this - value, then the vacuuming process is terminated. + During recoveries, if a node has not caused recovery failures + during the last grace period, any records of transgressions that + the node has caused recovery failures will be forgiven. This + resets the ban-counter back to zero for that node. @@ -470,76 +481,62 @@ RepackLimit Default: 10000 - During vacuuming, if the number of freelist records are more - than RepackLimit, then databases are - repacked to get rid of the freelist records to avoid - fragmentation. + During vacuuming, if the number of freelist records are more than + RepackLimit, then databases are repacked to + get rid of the freelist records to avoid fragmentation. - Databases are repacked only if both - RepackLimit and - VacuumLimit are exceeded. + Databases are repacked only if both RepackLimit + and VacuumLimit are exceeded. - VacuumLimit - Default: 5000 - - During vacuuming, if the number of deleted records are more - than VacuumLimit, then databases are - repacked to avoid fragmentation. - + RerecoveryTimeout + Default: 10 - Databases are repacked only if both - RepackLimit and - VacuumLimit are exceeded. + Once a recovery has completed, no additional recoveries are + permitted until this timeout has expired. - VacuumFastPathCount - Default: 60 + Samba3AvoidDeadlocks + Default: 0 + + Enable code that prevents deadlocks with Samba (only for Samba + 3.x). + - When a record is deleted, it is marked for deletion during - vacuuming. Vacuuming process usually processes this list to purge - the records from the database. If the number of records marked - for deletion are more than VacuumFastPathCount, then vacuuming - process will scan the complete database for empty records instead - of using the list of records marked for deletion. + This should be set to 1 when using Samba version 3.x to enable + special code in CTDB to avoid deadlock with Samba version 3.x. + This code is not required for Samba version 4.x and must not be + enabled for Samba 4.x. - DeferredAttachTO - Default: 120 + SeqnumInterval + Default: 1000 - When databases are frozen we do not allow clients to attach to the - databases. Instead of returning an error immediately to the application - the attach request from the client is deferred until the database - becomes available again at which stage we respond to the client. + Some databases have seqnum tracking enabled, so that samba will + be able to detect asynchronously when there has been updates + to the database. Everytime a database is updated its sequence + number is increased. - This timeout controls how long we will defer the request from the client - before timing it out and returning an error to the client. + This tunable is used to specify in 'ms' how frequently ctdb + will send out updates to remote nodes to inform them that the + sequence number is increased. - HopcountMakeSticky - Default: 50 - - If the database is set to 'STICKY' mode, using the 'ctdb setdbsticky' - command, any record that is seen as very hot and migrating so fast that - hopcount surpasses 50 is set to become a STICKY record for StickyDuration - seconds. This means that after each migration the record will be kept on - the node and prevented from being migrated off the node. - + StatHistoryInterval + Default: 1 - This setting allows one to try to identify such records and stop them from - migrating across the cluster so fast. This will improve performance for - certain workloads, such as locking.tdb if many clients are opening/closing - the same file concurrently. + Granularity of the statistics collected in the statistics + history. This is reported by 'ctdb stats' command. @@ -547,9 +544,9 @@ StickyDuration Default: 600 - Once a record has been found to be fetch-lock hot and has been flagged to - become STICKY, this is for how long, in seconds, the record will be - flagged as a STICKY record. + Once a record has been found to be fetch-lock hot and has been + flagged to become STICKY, this is for how long, in seconds, + the record will be flagged as a STICKY record. @@ -557,88 +554,97 @@ StickyPindown Default: 200 - Once a STICKY record has been migrated onto a node, it will be pinned down - on that node for this number of ms. Any request from other nodes to migrate - the record off the node will be deferred until the pindown timer expires. + Once a STICKY record has been migrated onto a node, it will be + pinned down on that node for this number of ms. Any request from + other nodes to migrate the record off the node will be deferred + until the pindown timer expires. - StatHistoryInterval - Default: 1 + TakeoverTimeout + Default: 9 - Granularity of the statistics collected in the statistics history. + This is how many seconds we allow controls to take for IP + failover events. - AllowClientDBAttach - Default: 1 + TickleUpdateInterval + Default: 20 - When set to 0, clients are not allowed to attach to any databases. - This can be used to temporarily block any new processes from attaching - to and accessing the databases. + How often will ctdb record and store the "tickle" information + used to kickstart stalled tcp connections after a recovery. - RecoverPDBBySeqNum - Default: 1 + TraverseTimeout + Default: 20 - When set to zero, database recovery for persistent databases - is record-by-record and recovery process simply collects the - most recent version of every individual record. + This setting controls how long we allow a traverse process to run. + After this timeout triggers, the main ctdb daemon will abort + the traverse if it has not yet finished. + + + + VacuumFastPathCount + Default: 60 - When set to non-zero, persistent databases will instead be - recovered as a whole db and not by individual records. The - node that contains the highest value stored in the record - "__db_sequence_number__" is selected and the copy of that - nodes database is used as the recovered database. + When a record is deleted, it is marked for deletion during + vacuuming. Vacuuming process usually processes this list to + purge the records from the database. If the number of records + marked for deletion are more than VacuumFastPathCount, then + vacuuming process will scan the complete database for empty + records instead of using the list of records marked for deletion. + + + + VacuumInterval + Default: 10 - By default, recovery of persistent databses is done using - __db_sequence_number__ record. + Periodic interval in seconds when vacuuming is triggered for + volatile databases. - FetchCollapse - Default: 1 + VacuumLimit + Default: 5000 - When many clients across many nodes try to access the same record at the - same time this can lead to a fetch storm where the record becomes very - active and bounces between nodes very fast. This leads to high CPU - utilization of the ctdbd daemon, trying to bounce that record around - very fast, and poor performance. + During vacuuming, if the number of deleted records are more than + VacuumLimit, then databases are repacked to + avoid fragmentation. - This parameter is used to activate a fetch-collapse. A fetch-collapse - is when we track which records we have requests in flight so that we only - keep one request in flight from a certain node, even if multiple smbd - processes are attemtping to fetch the record at the same time. This - can improve performance and reduce CPU utilization for certain - workloads. + Databases are repacked only if both RepackLimit + and VacuumLimit are exceeded. + + + + VacuumMaxRunTime + Default: 120 - This timeout controls if we should collapse multiple fetch operations - of the same record into a single request and defer all duplicates or not. + The maximum time in seconds for which the vacuuming process is + allowed to run. If vacuuming process takes longer than this + value, then the vacuuming process is terminated. - Samba3AvoidDeadlocks + VerboseMemoryNames Default: 0 - Enable code that prevents deadlocks with Samba (only for Samba 3.x). - - - This should be set to 1 when using Samba version 3.x to enable special - code in CTDB to avoid deadlock with Samba version 3.x. This code - is not required for Samba version 4.x and must not be enabled for - Samba 4.x. + This feature consumes additional memory. when used the talloc + library will create more verbose names for all talloc allocated + objects. + -- 2.5.5 From ce4e69c6aa812e68906f6ea2d5e4cd8d7181e2b9 Mon Sep 17 00:00:00 2001 From: Amitay Isaacs Date: Mon, 7 Mar 2016 19:26:43 +1100 Subject: [PATCH 05/29] ctdb-doc: Update tunables documentation Signed-off-by: Amitay Isaacs Reviewed-by: Martin Schwenke (cherry picked from commit e2539088e023ad88d314467a6a42b52c0decba77) --- ctdb/doc/ctdb-tunables.7.xml | 247 +++++++++++++++++++++---------------------- 1 file changed, 123 insertions(+), 124 deletions(-) diff --git a/ctdb/doc/ctdb-tunables.7.xml b/ctdb/doc/ctdb-tunables.7.xml index a6ef435..2c544fc 100644 --- a/ctdb/doc/ctdb-tunables.7.xml +++ b/ctdb/doc/ctdb-tunables.7.xml @@ -39,7 +39,8 @@ When set to 0, clients are not allowed to attach to any databases. This can be used to temporarily block any new processes from - attaching to and accessing the databases. + attaching to and accessing the databases. This is mainly used + for detaching a volatile database using 'ctdb detach'. @@ -65,8 +66,8 @@ DatabaseMaxDead Default: 5 - How many dead records per hashchain in the TDB database do we - allow before the freelist needs to be processed. + Maximum number of dead records per hash chain for the tdb databses + managed by ctdb. @@ -74,10 +75,9 @@ DBRecordCountWarn Default: 100000 - When set to non-zero, ctdb will log a warning when we try - to recover a database with more than this many records. This - will produce a warning if a database grows uncontrollably with - orphaned records. + When set to non-zero, ctdb will log a warning during recovery if + a database has more than this many records. This will produce a + warning if a database grows uncontrollably with orphaned records. @@ -85,10 +85,9 @@ DBRecordSizeWarn Default: 10000000 - When set to non-zero, ctdb will log a warning when we try to - recover a database where a single record is bigger than this. This - will produce a warning if a database record grows uncontrollably - with orphaned sub-records. + When set to non-zero, ctdb will log a warning during recovery + if a single record is bigger than this size. This will produce + a warning if a database record grows uncontrollably. @@ -96,8 +95,8 @@ DBSizeWarn Default: 1000000000 - When set to non-zero, ctdb will log a warning when we try to - recover a database bigger than this. This will produce a warning + When set to non-zero, ctdb will log a warning during recovery if + a database size is bigger than this. This will produce a warning if a database grows uncontrollably. @@ -108,7 +107,7 @@ When databases are frozen we do not allow clients to attach to the databases. Instead of returning an error immediately to the - application the attach request from the client is deferred until + client, the attach request from the client is deferred until the database becomes available again at which stage we respond to the client. @@ -122,10 +121,10 @@ DeterministicIPs Default: 0 - When enabled, this tunable makes ctdb try to keep public IP - addresses locked to specific nodes as far as possible. This makes - it easier for debugging since you can know that as long as all - nodes are healthy public IP X will always be hosted by node Y. + When set to 1, ctdb will try to keep public IP addresses locked + to specific nodes as far as possible. This makes it easier + for debugging since you can know that as long as all nodes are + healthy public IP X will always be hosted by node Y. The cost of using deterministic IP address assignment is that it @@ -140,15 +139,15 @@ DisableIPFailover Default: 0 - When enabled, ctdb will not perform failover or failback. Even - if a node fails while holding public IPs, ctdb will not recover - the IPs or assign them to another node. + When set to non-zero, ctdb will not perform failover or + failback. Even if a node fails while holding public IPs, ctdb + will not recover the IPs or assign them to another node. - When you enable this tunable, CTDB will no longer attempt + When this tunable is enabled, ctdb will no longer attempt to recover the cluster by failing IP addresses over to other nodes. This leads to a service outage until the administrator - has manually performed failover to replacement nodes using the + has manually performed IP failover to replacement nodes using the 'ctdb moveip' command. @@ -157,9 +156,10 @@ ElectionTimeout Default: 3 - When electing a new recovery master, this is how many seconds - we allow the election to take before we either deem the election - finished or we fail the election and start a new one. + The number of seconds to wait for the election of recovery + master to complete. If the election is not completed during this + interval, then that round of election fails and ctdb starts a + new election. @@ -167,7 +167,10 @@ EnableBans Default: 1 - When set to 0, this disables BANNING completely in the cluster + This parameter allows ctdb to ban a node if the node is misbehaving. + + + When set to 0, this disables banning completely in the cluster and thus nodes can not get banned, even it they break. Don't set to 0 unless you know what you are doing. You should set this to the same value on all nodes to avoid unexpected behaviour. @@ -194,24 +197,19 @@ FetchCollapse Default: 1 + This parameter is used to avoid multiple migration requests for + the same record from a single node. All the record requests for + the same record are queued up and processed when the record is + migrated to the current node. + + When many clients across many nodes try to access the same record at the same time this can lead to a fetch storm where the record becomes very active and bounces between nodes very fast. This leads to high CPU utilization of the ctdbd daemon, trying to bounce that record around very fast, and poor performance. - - - This parameter is used to activate a fetch-collapse. A - fetch-collapse is when we track which records we have requests in - flight so that we only keep one request in flight from a certain - node, even if multiple smbd processes are attemtping to fetch - the record at the same time. This can improve performance and - reduce CPU utilization for certain workloads. - - - This timeout controls if we should collapse multiple fetch - operations of the same record into a single request and defer - all duplicates or not. + This can improve performance and reduce CPU utilization for + certain workloads. @@ -219,18 +217,18 @@ HopcountMakeSticky Default: 50 - If the database is set to 'STICKY' mode, using the 'ctdb - setdbsticky' command, any record that is seen as very hot and - migrating so fast that hopcount surpasses 50 is set to become a - STICKY record for StickyDuration seconds. This means that after - each migration the record will be kept on the node and prevented - from being migrated off the node. + For database(s) marked STICKY (using 'ctdb setdbsticky'), + any record that is migrating so fast that hopcount + exceeds this limit is marked as STICKY record for + StickyDuration seconds. This means that + after each migration the sticky record will be kept on the node + StickyPindownmilliseconds and prevented from + being migrated off the node. - This setting allows one to try to identify such records and - stop them from migrating across the cluster so fast. This will - improve performance for certain workloads, such as locking.tdb - if many clients are opening/closing the same file concurrently. + This will improve performance for certain workloads, such as + locking.tdb if many clients are opening/closing the same file + concurrently. @@ -251,12 +249,13 @@ a node wait until marking the peer as DISCONNECTED. - If a node has hung, it can thus take - KeepaliveInterval*(KeepaliveLimit+1) seconds before we determine - that the node is DISCONNECTED and that we require a recovery. This - limitshould not be set too high since we want a hung node to be - detectec, and expunged from the cluster well before common CIFS - timeouts (45-90 seconds) kick in. + If a node has hung, it can take + KeepaliveInterval * + (KeepaliveLimit + 1) seconds before + ctdb determines that the node is DISCONNECTED and performs + a recovery. This limit should not be set too high to enable + early detection and avoid any application timeouts (e.g. SMB1) + to kick in before the fail over is completed. @@ -264,8 +263,7 @@ LCP2PublicIPs Default: 1 - When enabled this switches ctdb to use the LCP2 ip allocation - algorithm. + When set to 1, ctdb uses the LCP2 ip allocation algorithm. @@ -273,11 +271,11 @@ LogLatencyMs Default: 0 - When set to non-zero, this will make the main daemon log any - operation that took longer than this value, in 'ms', to complete. - These include "how long time a lockwait child process needed", - "how long time to write to a persistent database" but also "how - long did it take to get a response to a CALL from a remote node". + When set to non-zero, ctdb will log if certains operations + take longer than this value, in milliseconds, to complete. + These operations include "process a record request from client", + "take a record or database lock", "update a persistent database + record" and "vaccum a database". @@ -304,7 +302,7 @@ MonitorInterval Default: 15 - How often should ctdb run the event scripts in seconds to check + How often should ctdb run the 'monitor' event in seconds to check for a node's health. @@ -313,7 +311,7 @@ MonitorTimeoutCount Default: 20 - How many monitor events in a row need to timeout before a node + How many 'monitor' events in a row need to timeout before a node is flagged as UNHEALTHY. This setting is useful if scripts can not be written so that they do not hang for benign reasons. @@ -324,9 +322,9 @@ Default: 0 When set to 1, ctdb will not perform failback of IP addresses - when a node becomes healthy. Ctdb WILL perform failover of public - IP addresses when a node becomes UNHEALTHY, but when the node - becomes HEALTHY again, ctdb will not fail the addresses back. + when a node becomes healthy. When a node becomes UNHEALTHY, + ctdb WILL perform failover of public IP addresses, but when the + node becomes HEALTHY again, ctdb will not fail the addresses back. Use with caution! Normally when a node becomes available to the @@ -336,7 +334,7 @@ approximately the same number of public addresses it hosts. - When you enable this tunable, CTDB will no longer attempt to + When you enable this tunable, ctdb will no longer attempt to rebalance the cluster by failing IP addresses back to the new nodes. An unbalanced cluster will therefore remain unbalanced until there is manual intervention from the administrator. When @@ -349,11 +347,11 @@ NoIPHostOnAllDisabled Default: 0 - If no nodes are healthy then by default ctdb will happily host + If no nodes are HEALTHY then by default ctdb will happily host public IPs on disabled (unhealthy or administratively disabled) nodes. This can cause problems, for example if the underlying cluster filesystem is not mounted. When set to 1 on a node and - that node is disabled it, any IPs hosted by this node will be + that node is disabled, any IPs hosted by this node will be released and the node will not takeover any IPs until it is no longer disabled. @@ -386,7 +384,9 @@ If the main dameon has not heard a "ping" from the recovery dameon for this many seconds, the main dameon will log a message that - the recovery daemon is potentially hung. + the recovery daemon is potentially hung. This also increments a + counter which is checked against RecdFailCount + for detection of hung recovery daemon. @@ -397,7 +397,7 @@ When using a reclock file for split brain prevention, if set to non-zero this tunable will make the recovery dameon log a message if the fcntl() call to lock/testlock the recovery file - takes longer than this number of ms. + takes longer than this number of milliseconds. @@ -405,9 +405,8 @@ RecoverInterval Default: 1 - How frequently in seconds should the recovery daemon perform - the consistency checks that determine if we need to perform a - recovery or not. + How frequently in seconds should the recovery daemon perform the + consistency checks to determine if it should perform a recovery. @@ -434,7 +433,7 @@ RecoverTimeout - Default: 20 + Default: 120 This is the default setting for timeouts for controls when sent from the recovery daemon. We allow longer control timeouts from @@ -448,12 +447,14 @@ RecoveryBanPeriod Default: 300 - If a node becomes banned causing repetitive recovery failures. The - node will eventually become banned from the cluster. This - controls how long the culprit node will be banned from the - cluster before it is allowed to try to join the cluster again. - Don't set to small. A node gets banned for a reason and it is - usually due to real problems with the node. + The duration in seconds for which a node is banned if the node + fails during recovery. After this time has elapsed the node will + automatically get unbanned and will attempt to rejoin the cluster. + + + A node usually gets banned due to real problems with the node. + Don't set this value too small. Otherwise, a problematic node + will try to re-join cluster too soon causing unnecessary recoveries. @@ -461,8 +462,9 @@ RecoveryDropAllIPs Default: 120 - If we have been stuck in recovery, or stopped, or banned, mode for - this many seconds we will force drop all held public addresses. + If a node is stuck in recovery, or stopped, or banned, for this + many seconds, then ctdb will release all public addresses on + that node. @@ -471,9 +473,9 @@ Default: 120 During recoveries, if a node has not caused recovery failures - during the last grace period, any records of transgressions that - the node has caused recovery failures will be forgiven. This - resets the ban-counter back to zero for that node. + during the last grace period in seconds, any records of + transgressions that the node has caused recovery failures will be + forgiven. This resets the ban-counter back to zero for that node. @@ -482,8 +484,8 @@ Default: 10000 During vacuuming, if the number of freelist records are more than - RepackLimit, then databases are repacked to - get rid of the freelist records to avoid fragmentation. + RepackLimit, then the database is repacked + to get rid of the freelist records to avoid fragmentation. Databases are repacked only if both RepackLimit @@ -496,7 +498,7 @@ Default: 10 Once a recovery has completed, no additional recoveries are - permitted until this timeout has expired. + permitted until this timeout in seconds has expired. @@ -504,14 +506,13 @@ Samba3AvoidDeadlocks Default: 0 - Enable code that prevents deadlocks with Samba (only for Samba - 3.x). - - - This should be set to 1 when using Samba version 3.x to enable - special code in CTDB to avoid deadlock with Samba version 3.x. - This code is not required for Samba version 4.x and must not be - enabled for Samba 4.x. + If set to non-zero, enable code that prevents deadlocks with Samba + (only for Samba 3.x). + + This should be set to 1 only when using Samba version 3.x + to enable special code in ctdb to avoid deadlock with Samba + version 3.x. This code is not required for Samba version 4.x + and must not be enabled for Samba 4.x. @@ -525,9 +526,9 @@ number is increased. - This tunable is used to specify in 'ms' how frequently ctdb - will send out updates to remote nodes to inform them that the - sequence number is increased. + This tunable is used to specify in milliseconds how frequently + ctdb will send out updates to remote nodes to inform them that + the sequence number is increased. @@ -544,9 +545,8 @@ StickyDuration Default: 600 - Once a record has been found to be fetch-lock hot and has been - flagged to become STICKY, this is for how long, in seconds, - the record will be flagged as a STICKY record. + Once a record has been marked STICKY, this is the duration in + seconds, the record will be flagged as a STICKY record. @@ -555,9 +555,9 @@ Default: 200 Once a STICKY record has been migrated onto a node, it will be - pinned down on that node for this number of ms. Any request from - other nodes to migrate the record off the node will be deferred - until the pindown timer expires. + pinned down on that node for this number of milliseconds. Any + request from other nodes to migrate the record off the node will + be deferred. @@ -565,8 +565,8 @@ TakeoverTimeout Default: 9 - This is how many seconds we allow controls to take for IP - failover events. + This is the duration in seconds in which ctdb tries to complete IP + failover. @@ -574,8 +574,8 @@ TickleUpdateInterval Default: 20 - How often will ctdb record and store the "tickle" information - used to kickstart stalled tcp connections after a recovery. + Every TickleUpdateInterval seconds, ctdb + synchronizes the client connection information across nodes. @@ -583,9 +583,9 @@ TraverseTimeout Default: 20 - This setting controls how long we allow a traverse process to run. - After this timeout triggers, the main ctdb daemon will abort - the traverse if it has not yet finished. + This is the duration in seconds for which a database traverse + is allowed to run. If the traverse does not complete during + this interval, ctdb will abort the traverse. @@ -593,12 +593,11 @@ VacuumFastPathCount Default: 60 - When a record is deleted, it is marked for deletion during - vacuuming. Vacuuming process usually processes this list to - purge the records from the database. If the number of records - marked for deletion are more than VacuumFastPathCount, then - vacuuming process will scan the complete database for empty - records instead of using the list of records marked for deletion. + During a vacuuming run, ctdb usually processes only the records + marked for deletion also called the fast path vacuuming. After + finishing VacuumFastPathCount number of fast + path vacuuming runs, ctdb will trigger a scan of complete database + for any empty records that need to be deleted. @@ -639,9 +638,9 @@ VerboseMemoryNames Default: 0 - This feature consumes additional memory. when used the talloc - library will create more verbose names for all talloc allocated - objects. + When set to non-zero, ctdb assigns verbose names for some of + the talloc allocated memory objects. These names are visible + in the talloc memory report generated by 'ctdb dumpmemory'. -- 2.5.5 From 911a9ea454004487f888a96bc3de8d9ccfdbf541 Mon Sep 17 00:00:00 2001 From: Amitay Isaacs Date: Tue, 8 Mar 2016 14:37:41 +1100 Subject: [PATCH 06/29] ctdb-doc: Add documentation for missing tunables Signed-off-by: Amitay Isaacs Reviewed-by: Martin Schwenke (cherry picked from commit 3680aba2126d967a5518ab2d35d9e6d3f338e75e) --- ctdb/doc/ctdb-tunables.7.xml | 64 ++++++++++++++++++++++++++++++++++---------- 1 file changed, 50 insertions(+), 14 deletions(-) diff --git a/ctdb/doc/ctdb-tunables.7.xml b/ctdb/doc/ctdb-tunables.7.xml index 2c544fc..591dcf5 100644 --- a/ctdb/doc/ctdb-tunables.7.xml +++ b/ctdb/doc/ctdb-tunables.7.xml @@ -45,6 +45,16 @@ + AllowUnhealthyDBRead + Default: 0 + + When set to 1, ctdb allows database traverses to read unhealthy + databases. By default, ctdb does not allow reading records from + unhealthy databases. + + + + ControlTimeout Default: 60 @@ -268,6 +278,17 @@ + LockProcessesPerDB + Default: 200 + + This is the maximum number of lock helper processes ctdb will + create for obtaining record locks. When ctdb cannot get a record + lock without blocking, it creates a helper process that waits + for the lock to be obtained. + + + + LogLatencyMs Default: 0 @@ -280,21 +301,12 @@ - MaxRedirectCount - Default: 3 + MaxQueueDropMsg + Default: 1000000 - If we are not the DMASTER and need to fetch a record across the - network we first send the request to the LMASTER after which the - record is passed onto the current DMASTER. If the DMASTER changes - before the request has reached that node, the request will be - passed onto the "next" DMASTER. For very hot records that migrate - rapidly across the cluster this can cause a request to "chase" - the record for many hops before it catches up with the record. - - - When chasing a record, this is how many hops we will chase - the record for before going back to the LMASTER to ask for - new guidance. + This is the maximum number of messages to be queued up for + a client before ctdb will treat the client as hung and will + terminate the client connection. @@ -369,6 +381,19 @@ + PullDBPreallocation + Default: 10*1024*1024 + + This is the size of a record buffer to pre-allocate for sending + reply to PULLDB control. Usually record buffer starts with size + of the first record and gets reallocated every time a new record + is added to the record buffer. For a large number of records, + this can be very inefficient to grow the record buffer one record + at a time. + + + + RecdFailCount Default: 10 @@ -571,6 +596,17 @@ + TDBMutexEnabled + Default: 0 + + This paramter enables TDB_MUTEX_LOCKING feature on volatile + databases if the robust mutexes are supported. This optimizes the + record locking using robust mutexes and is much more efficient + that using posix locks. + + + + TickleUpdateInterval Default: 20 -- 2.5.5 From ae6922ff2f1dad412dc73658509d64a5e752a78b Mon Sep 17 00:00:00 2001 From: Amitay Isaacs Date: Tue, 8 Mar 2016 14:51:59 +1100 Subject: [PATCH 07/29] ctdb-recovery-helper: Get tunables first, so control timeout can be set During the recovery process, the timeout value for sending all controls is decided by RecoverTimeout tunable. So in the recovery process, first get the tunables, so the control timeout gets set correctly. Signed-off-by: Amitay Isaacs Reviewed-by: Martin Schwenke (cherry picked from commit 700f39372ad9c893f8eb8884cd24999dc7026e60) --- ctdb/server/ctdb_recovery_helper.c | 97 ++++++++++++++++++++------------------ 1 file changed, 51 insertions(+), 46 deletions(-) diff --git a/ctdb/server/ctdb_recovery_helper.c b/ctdb/server/ctdb_recovery_helper.c index f0fa12d..607faa6 100644 --- a/ctdb/server/ctdb_recovery_helper.c +++ b/ctdb/server/ctdb_recovery_helper.c @@ -34,7 +34,9 @@ #include "protocol/protocol_api.h" #include "client/client.h" -#define TIMEOUT() timeval_current_ofs(10, 0) +static int recover_timeout = 120; + +#define TIMEOUT() timeval_current_ofs(recover_timeout, 0) static void LOG(const char *fmt, ...) { @@ -1215,10 +1217,10 @@ static bool db_recovery_recv(struct tevent_req *req, int *count) /* * Run the parallel database recovery * + * - Get tunables * - Get nodemap * - Get vnnmap * - Get capabilities from all nodes - * - Get tunables from all nodes * - Get dbmap * - Set RECOVERY_ACTIVE * - Send START_RECOVERY @@ -1242,10 +1244,10 @@ struct recovery_state { struct ctdb_dbid_map *dbmap; }; +static void recovery_tunables_done(struct tevent_req *subreq); static void recovery_nodemap_done(struct tevent_req *subreq); static void recovery_vnnmap_done(struct tevent_req *subreq); static void recovery_capabilities_done(struct tevent_req *subreq); -static void recovery_tunables_done(struct tevent_req *subreq); static void recovery_dbmap_done(struct tevent_req *subreq); static void recovery_active_done(struct tevent_req *subreq); static void recovery_start_recovery_done(struct tevent_req *subreq); @@ -1273,17 +1275,59 @@ static struct tevent_req *recovery_send(TALLOC_CTX *mem_ctx, state->generation = generation; state->destnode = ctdb_client_pnn(client); - ctdb_req_control_get_nodemap(&request); - subreq = ctdb_client_control_send(mem_ctx, ev, client, state->destnode, - TIMEOUT(), &request); + ctdb_req_control_get_all_tunables(&request); + subreq = ctdb_client_control_send(state, state->ev, state->client, + state->destnode, TIMEOUT(), + &request); if (tevent_req_nomem(subreq, req)) { return tevent_req_post(req, ev); } - tevent_req_set_callback(subreq, recovery_nodemap_done, req); + tevent_req_set_callback(subreq, recovery_tunables_done, req); return req; } +static void recovery_tunables_done(struct tevent_req *subreq) +{ + struct tevent_req *req = tevent_req_callback_data( + subreq, struct tevent_req); + struct recovery_state *state = tevent_req_data( + req, struct recovery_state); + struct ctdb_reply_control *reply; + struct ctdb_req_control request; + int ret; + bool status; + + status = ctdb_client_control_recv(subreq, &ret, state, &reply); + TALLOC_FREE(subreq); + if (! status) { + LOG("control GET_ALL_TUNABLES failed, ret=%d\n", ret); + tevent_req_error(req, ret); + return; + } + + ret = ctdb_reply_control_get_all_tunables(reply, state, + &state->tun_list); + if (ret != 0) { + LOG("control GET_ALL_TUNABLES failed, ret=%d\n", ret); + tevent_req_error(req, EPROTO); + return; + } + + talloc_free(reply); + + recover_timeout = state->tun_list->recover_timeout; + + ctdb_req_control_get_nodemap(&request); + subreq = ctdb_client_control_send(state, state->ev, state->client, + state->destnode, TIMEOUT(), + &request); + if (tevent_req_nomem(subreq, req)) { + return; + } + tevent_req_set_callback(subreq, recovery_nodemap_done, req); +} + static void recovery_nodemap_done(struct tevent_req *subreq) { struct tevent_req *req = tevent_req_callback_data( @@ -1420,45 +1464,6 @@ static void recovery_capabilities_done(struct tevent_req *subreq) talloc_free(reply); - ctdb_req_control_get_all_tunables(&request); - subreq = ctdb_client_control_send(state, state->ev, state->client, - state->destnode, TIMEOUT(), - &request); - if (tevent_req_nomem(subreq, req)) { - return; - } - tevent_req_set_callback(subreq, recovery_tunables_done, req); -} - -static void recovery_tunables_done(struct tevent_req *subreq) -{ - struct tevent_req *req = tevent_req_callback_data( - subreq, struct tevent_req); - struct recovery_state *state = tevent_req_data( - req, struct recovery_state); - struct ctdb_reply_control *reply; - struct ctdb_req_control request; - int ret; - bool status; - - status = ctdb_client_control_recv(subreq, &ret, state, &reply); - TALLOC_FREE(subreq); - if (! status) { - LOG("control GET_ALL_TUNABLES failed, ret=%d\n", ret); - tevent_req_error(req, ret); - return; - } - - ret = ctdb_reply_control_get_all_tunables(reply, state, - &state->tun_list); - if (ret != 0) { - LOG("control GET_ALL_TUNABLES failed, ret=%d\n", ret); - tevent_req_error(req, EPROTO); - return; - } - - talloc_free(reply); - ctdb_req_control_get_dbmap(&request); subreq = ctdb_client_control_send(state, state->ev, state->client, state->destnode, TIMEOUT(), -- 2.5.5 From f29700a559c94329cb97c082707babbca4377c9f Mon Sep 17 00:00:00 2001 From: Amitay Isaacs Date: Fri, 26 Feb 2016 14:12:19 +1100 Subject: [PATCH 08/29] ctdb-client: Add client API for sending message to multiple nodes Signed-off-by: Amitay Isaacs Reviewed-by: Martin Schwenke (cherry picked from commit 67799c73af486962b897d1841a8a4234fdee824b) --- ctdb/client/client.h | 10 ++++ ctdb/client/client_message.c | 132 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 142 insertions(+) diff --git a/ctdb/client/client.h b/ctdb/client/client.h index 48b69b6..66be313 100644 --- a/ctdb/client/client.h +++ b/ctdb/client/client.h @@ -72,6 +72,16 @@ struct tevent_req *ctdb_client_message_send(TALLOC_CTX *mem_ctx, bool ctdb_client_message_recv(struct tevent_req *req, int *perr); +struct tevent_req *ctdb_client_message_multi_send( + TALLOC_CTX *mem_ctx, + struct tevent_context *ev, + struct ctdb_client_context *client, + uint32_t *pnn_list, int count, + struct ctdb_req_message *message); + +bool ctdb_client_message_multi_recv(struct tevent_req *req, int *perr, + TALLOC_CTX *mem_ctx, int **perr_list); + int ctdb_client_message(TALLOC_CTX *mem_ctx, struct tevent_context *ev, struct ctdb_client_context *client, uint32_t destnode, struct ctdb_req_message *message); diff --git a/ctdb/client/client_message.c b/ctdb/client/client_message.c index c316f42..8c75548 100644 --- a/ctdb/client/client_message.c +++ b/ctdb/client/client_message.c @@ -157,6 +157,138 @@ void ctdb_client_req_message(struct ctdb_client_context *client, } /* + * Handle multiple nodes + */ + +struct ctdb_client_message_multi_state { + uint32_t *pnn_list; + int count; + int done; + int err; + int *err_list; +}; + +struct message_index_state { + struct tevent_req *req; + int index; +}; + +static void ctdb_client_message_multi_done(struct tevent_req *subreq); + +struct tevent_req *ctdb_client_message_multi_send( + TALLOC_CTX *mem_ctx, + struct tevent_context *ev, + struct ctdb_client_context *client, + uint32_t *pnn_list, int count, + struct ctdb_req_message *message) +{ + struct tevent_req *req, *subreq; + struct ctdb_client_message_multi_state *state; + int i; + + if (pnn_list == NULL || count == 0) { + return NULL; + } + + req = tevent_req_create(mem_ctx, &state, + struct ctdb_client_message_multi_state); + if (req == NULL) { + return NULL; + } + + state->pnn_list = pnn_list; + state->count = count; + state->done = 0; + state->err = 0; + state->err_list = talloc_zero_array(state, int, count); + if (tevent_req_nomem(state->err_list, req)) { + return tevent_req_post(req, ev); + } + + for (i=0; ireq = req; + substate->index = i; + + tevent_req_set_callback(subreq, ctdb_client_message_multi_done, + substate); + } + + return req; +} + +static void ctdb_client_message_multi_done(struct tevent_req *subreq) +{ + struct message_index_state *substate = tevent_req_callback_data( + subreq, struct message_index_state); + struct tevent_req *req = substate->req; + int idx = substate->index; + struct ctdb_client_message_multi_state *state = tevent_req_data( + req, struct ctdb_client_message_multi_state); + bool status; + int ret; + + status = ctdb_client_message_recv(subreq, &ret); + TALLOC_FREE(subreq); + if (! status) { + if (state->err == 0) { + state->err = ret; + state->err_list[idx] = state->err; + } + } + + state->done += 1; + + if (state->done == state->count) { + tevent_req_done(req); + } +} + +bool ctdb_client_message_multi_recv(struct tevent_req *req, int *perr, + TALLOC_CTX *mem_ctx, int **perr_list) +{ + struct ctdb_client_message_multi_state *state = tevent_req_data( + req, struct ctdb_client_message_multi_state); + int err; + + if (tevent_req_is_unix_error(req, &err)) { + if (perr != NULL) { + *perr = err; + } + if (perr_list != NULL) { + *perr_list = talloc_steal(mem_ctx, state->err_list); + } + return false; + } + + if (perr != NULL) { + *perr = state->err; + } + + if (perr_list != NULL) { + *perr_list = talloc_steal(mem_ctx, state->err_list); + } + + if (state->err != 0) { + return false; + } + + return true; +} + +/* * sync version of message send */ -- 2.5.5 From 1391f8e356a7663814bf94b2fb8109d05db15890 Mon Sep 17 00:00:00 2001 From: Amitay Isaacs Date: Tue, 8 Mar 2016 17:29:31 +1100 Subject: [PATCH 09/29] ctdb-tunables: Add new tunable RecBufferSizeLimit This will be used to limit the size of record buffer sent in newer controls for recovery and existing controls for vacuuming. Signed-off-by: Amitay Isaacs Reviewed-by: Martin Schwenke (cherry picked from commit c41808e6d292ec4cd861af545478e7dfb5c448e8) --- ctdb/doc/ctdb-tunables.7.xml | 10 ++++++++++ ctdb/protocol/protocol.h | 1 + ctdb/server/ctdb_tunables.c | 1 + 3 files changed, 12 insertions(+) diff --git a/ctdb/doc/ctdb-tunables.7.xml b/ctdb/doc/ctdb-tunables.7.xml index 591dcf5..d2d2616 100644 --- a/ctdb/doc/ctdb-tunables.7.xml +++ b/ctdb/doc/ctdb-tunables.7.xml @@ -394,6 +394,16 @@ + RecBufferSizeLimit + Default: 1000000 + + This is the limit on the size of the record buffer to be sent + in various controls. This limit is used by new controls used + for recovery and controls used in vacuuming. + + + + RecdFailCount Default: 10 diff --git a/ctdb/protocol/protocol.h b/ctdb/protocol/protocol.h index a2a0c45..28d4c41 100644 --- a/ctdb/protocol/protocol.h +++ b/ctdb/protocol/protocol.h @@ -619,6 +619,7 @@ struct ctdb_tunable_list { uint32_t samba3_hack; uint32_t mutex_enabled; uint32_t lock_processes_per_db; + uint32_t rec_buffer_size_limit; }; struct ctdb_tickle_list { diff --git a/ctdb/server/ctdb_tunables.c b/ctdb/server/ctdb_tunables.c index ba184e7..50c5c77 100644 --- a/ctdb/server/ctdb_tunables.c +++ b/ctdb/server/ctdb_tunables.c @@ -93,6 +93,7 @@ static const struct { { "Samba3AvoidDeadlocks", 0, offsetof(struct ctdb_tunable_list, samba3_hack), false }, { "TDBMutexEnabled", 0, offsetof(struct ctdb_tunable_list, mutex_enabled), false }, { "LockProcessesPerDB", 200, offsetof(struct ctdb_tunable_list, lock_processes_per_db), false }, + { "RecBufferSizeLimit", 1000000, offsetof(struct ctdb_tunable_list, rec_buffer_size_limit), false }, }; /* -- 2.5.5 From 8c6892f0a23893107874f9a4a470ce8fcee4ae72 Mon Sep 17 00:00:00 2001 From: Amitay Isaacs Date: Fri, 19 Feb 2016 10:45:19 +1100 Subject: [PATCH 10/29] ctdb-protocol: Add new data type ctdb_pulldb_ext for new control Signed-off-by: Amitay Isaacs Reviewed-by: Martin Schwenke (cherry picked from commit fe69b72569494ef09dc8fa8673af02c0bfc18434) --- ctdb/protocol/protocol.h | 6 ++++++ ctdb/protocol/protocol_private.h | 5 +++++ ctdb/protocol/protocol_types.c | 28 ++++++++++++++++++++++++++++ 3 files changed, 39 insertions(+) diff --git a/ctdb/protocol/protocol.h b/ctdb/protocol/protocol.h index 28d4c41..a44e768 100644 --- a/ctdb/protocol/protocol.h +++ b/ctdb/protocol/protocol.h @@ -457,6 +457,12 @@ struct ctdb_pulldb { uint32_t lmaster; }; +struct ctdb_pulldb_ext { + uint32_t db_id; + uint32_t lmaster; + uint64_t srvid; +}; + #define CTDB_RECOVERY_NORMAL 0 #define CTDB_RECOVERY_ACTIVE 1 diff --git a/ctdb/protocol/protocol_private.h b/ctdb/protocol/protocol_private.h index 65193d9..a67d622 100644 --- a/ctdb/protocol/protocol_private.h +++ b/ctdb/protocol/protocol_private.h @@ -91,6 +91,11 @@ void ctdb_pulldb_push(struct ctdb_pulldb *pulldb, uint8_t *buf); int ctdb_pulldb_pull(uint8_t *buf, size_t buflen, TALLOC_CTX *mem_ctx, struct ctdb_pulldb **out); +size_t ctdb_pulldb_ext_len(struct ctdb_pulldb_ext *pulldb); +void ctdb_pulldb_ext_push(struct ctdb_pulldb_ext *pulldb, uint8_t *buf); +int ctdb_pulldb_ext_pull(uint8_t *buf, size_t buflen, TALLOC_CTX *mem_ctx, + struct ctdb_pulldb_ext **out); + size_t ctdb_traverse_start_len(struct ctdb_traverse_start *traverse); void ctdb_traverse_start_push(struct ctdb_traverse_start *traverse, uint8_t *buf); diff --git a/ctdb/protocol/protocol_types.c b/ctdb/protocol/protocol_types.c index e36d2e8..a026333 100644 --- a/ctdb/protocol/protocol_types.c +++ b/ctdb/protocol/protocol_types.c @@ -473,6 +473,34 @@ int ctdb_pulldb_pull(uint8_t *buf, size_t buflen, TALLOC_CTX *mem_ctx, return 0; } +size_t ctdb_pulldb_ext_len(struct ctdb_pulldb_ext *pulldb) +{ + return sizeof(struct ctdb_pulldb_ext); +} + +void ctdb_pulldb_ext_push(struct ctdb_pulldb_ext *pulldb, uint8_t *buf) +{ + memcpy(buf, pulldb, sizeof(struct ctdb_pulldb_ext)); +} + +int ctdb_pulldb_ext_pull(uint8_t *buf, size_t buflen, TALLOC_CTX *mem_ctx, + struct ctdb_pulldb_ext **out) +{ + struct ctdb_pulldb_ext *pulldb; + + if (buflen < sizeof(struct ctdb_pulldb_ext)) { + return EMSGSIZE; + } + + pulldb = talloc_memdup(mem_ctx, buf, sizeof(struct ctdb_pulldb_ext)); + if (pulldb == NULL) { + return ENOMEM; + } + + *out = pulldb; + return 0; +} + size_t ctdb_ltdb_header_len(struct ctdb_ltdb_header *header) { return sizeof(struct ctdb_ltdb_header); -- 2.5.5 From 25f1f205c76964e2a51487c1d8128972f606f983 Mon Sep 17 00:00:00 2001 From: Amitay Isaacs Date: Fri, 19 Feb 2016 10:54:15 +1100 Subject: [PATCH 11/29] ctdb-protocol: Add new controls DB_PULL and DB_PUSH_START/DB_PUSH_CONFIRM Signed-off-by: Amitay Isaacs Reviewed-by: Martin Schwenke (cherry picked from commit 0fd156ae8478f2da35a68c6bd3f90f50a06aecb8) --- ctdb/protocol/protocol.h | 5 +++ ctdb/protocol/protocol_api.h | 14 +++++++ ctdb/protocol/protocol_client.c | 70 +++++++++++++++++++++++++++++++++++ ctdb/protocol/protocol_control.c | 68 ++++++++++++++++++++++++++++++++++ ctdb/tests/src/protocol_client_test.c | 48 ++++++++++++++++++++++++ ctdb/tests/src/protocol_types_test.c | 18 +++++++++ 6 files changed, 223 insertions(+) diff --git a/ctdb/protocol/protocol.h b/ctdb/protocol/protocol.h index a44e768..16d1801 100644 --- a/ctdb/protocol/protocol.h +++ b/ctdb/protocol/protocol.h @@ -355,6 +355,9 @@ enum ctdb_controls {CTDB_CONTROL_PROCESS_EXISTS = 0, CTDB_CONTROL_DB_TRANSACTION_START = 143, CTDB_CONTROL_DB_TRANSACTION_COMMIT = 144, CTDB_CONTROL_DB_TRANSACTION_CANCEL = 145, + CTDB_CONTROL_DB_PULL = 146, + CTDB_CONTROL_DB_PUSH_START = 147, + CTDB_CONTROL_DB_PUSH_CONFIRM = 148, }; #define CTDB_MONITORING_ACTIVE 0 @@ -862,6 +865,7 @@ struct ctdb_req_control_data { struct ctdb_vnn_map *vnnmap; uint32_t loglevel; struct ctdb_pulldb *pulldb; + struct ctdb_pulldb_ext *pulldb_ext; struct ctdb_rec_buffer *recbuf; uint32_t recmode; const char *db_name; @@ -930,6 +934,7 @@ struct ctdb_reply_control_data { struct ctdb_uint8_array *u8_array; struct ctdb_db_statistics *dbstats; enum ctdb_runstate runstate; + uint32_t num_records; } data; }; diff --git a/ctdb/protocol/protocol_api.h b/ctdb/protocol/protocol_api.h index 6fb9cec..ed7bdcb 100644 --- a/ctdb/protocol/protocol_api.h +++ b/ctdb/protocol/protocol_api.h @@ -634,6 +634,20 @@ void ctdb_req_control_db_transaction_cancel(struct ctdb_req_control *request, uint32_t db_id); int ctdb_reply_control_db_transaction_cancel(struct ctdb_reply_control *reply); +void ctdb_req_control_db_pull(struct ctdb_req_control *request, + struct ctdb_pulldb_ext *pulldb_ext); +int ctdb_reply_control_db_pull(struct ctdb_reply_control *reply, + uint32_t *num_records); + +void ctdb_req_control_db_push_start(struct ctdb_req_control *request, + struct ctdb_pulldb_ext *pulldb_ext); +int ctdb_reply_control_db_push_start(struct ctdb_reply_control *reply); + +void ctdb_req_control_db_push_confirm(struct ctdb_req_control *request, + uint32_t db_id); +int ctdb_reply_control_db_push_confirm(struct ctdb_reply_control *reply, + uint32_t *num_records); + /* From protocol/protocol_message.c */ int ctdb_req_message_push(struct ctdb_req_header *h, diff --git a/ctdb/protocol/protocol_client.c b/ctdb/protocol/protocol_client.c index a4c19d9..09cd7e9 100644 --- a/ctdb/protocol/protocol_client.c +++ b/ctdb/protocol/protocol_client.c @@ -2468,3 +2468,73 @@ int ctdb_reply_control_db_transaction_cancel(struct ctdb_reply_control *reply) { return ctdb_reply_control_generic(reply); } + +/* CTDB_CONTROL_DB_PULL */ + +void ctdb_req_control_db_pull(struct ctdb_req_control *request, + struct ctdb_pulldb_ext *pulldb_ext) +{ + request->opcode = CTDB_CONTROL_DB_PULL; + request->pad = 0; + request->srvid = 0; + request->client_id = 0; + request->flags = 0; + + request->rdata.opcode = CTDB_CONTROL_DB_PULL; + request->rdata.data.pulldb_ext = pulldb_ext; +} + +int ctdb_reply_control_db_pull(struct ctdb_reply_control *reply, + uint32_t *num_records) +{ + if (reply->status == 0 && + reply->rdata.opcode == CTDB_CONTROL_DB_PULL) { + *num_records = reply->rdata.data.num_records; + } + return reply->status; +} + +/* CTDB_CONTROL_DB_PUSH_START */ + +void ctdb_req_control_db_push_start(struct ctdb_req_control *request, + struct ctdb_pulldb_ext *pulldb_ext) +{ + request->opcode = CTDB_CONTROL_DB_PUSH_START; + request->pad = 0; + request->srvid = 0; + request->client_id = 0; + request->flags = 0; + + request->rdata.opcode = CTDB_CONTROL_DB_PUSH_START; + request->rdata.data.pulldb_ext = pulldb_ext; +} + +int ctdb_reply_control_db_push_start(struct ctdb_reply_control *reply) +{ + return ctdb_reply_control_generic(reply); +} + +/* CTDB_CONTROL_DB_PUSH_CONFIRM */ + +void ctdb_req_control_db_push_confirm(struct ctdb_req_control *request, + uint32_t db_id) +{ + request->opcode = CTDB_CONTROL_DB_PUSH_CONFIRM; + request->pad = 0; + request->srvid = 0; + request->client_id = 0; + request->flags = 0; + + request->rdata.opcode = CTDB_CONTROL_DB_PUSH_CONFIRM; + request->rdata.data.db_id = db_id; +} + +int ctdb_reply_control_db_push_confirm(struct ctdb_reply_control *reply, + uint32_t *num_records) +{ + if (reply->status == 0 && + reply->rdata.opcode == CTDB_CONTROL_DB_PUSH_CONFIRM) { + *num_records = reply->rdata.data.num_records; + } + return reply->status; +} diff --git a/ctdb/protocol/protocol_control.c b/ctdb/protocol/protocol_control.c index 1c8364e..e7487da 100644 --- a/ctdb/protocol/protocol_control.c +++ b/ctdb/protocol/protocol_control.c @@ -481,6 +481,18 @@ static size_t ctdb_req_control_data_len(struct ctdb_req_control_data *cd) case CTDB_CONTROL_DB_TRANSACTION_CANCEL: len = ctdb_uint32_len(cd->data.db_id); break; + + case CTDB_CONTROL_DB_PULL: + len = ctdb_pulldb_ext_len(cd->data.pulldb_ext); + break; + + case CTDB_CONTROL_DB_PUSH_START: + len = ctdb_pulldb_ext_len(cd->data.pulldb_ext); + break; + + case CTDB_CONTROL_DB_PUSH_CONFIRM: + len = ctdb_uint32_len(cd->data.db_id); + break; } return len; @@ -791,6 +803,18 @@ static void ctdb_req_control_data_push(struct ctdb_req_control_data *cd, case CTDB_CONTROL_DB_TRANSACTION_CANCEL: ctdb_uint32_push(cd->data.db_id, buf); break; + + case CTDB_CONTROL_DB_PULL: + ctdb_pulldb_ext_push(cd->data.pulldb_ext, buf); + break; + + case CTDB_CONTROL_DB_PUSH_START: + ctdb_pulldb_ext_push(cd->data.pulldb_ext, buf); + break; + + case CTDB_CONTROL_DB_PUSH_CONFIRM: + ctdb_uint32_push(cd->data.db_id, buf); + break; } } @@ -1177,6 +1201,21 @@ static int ctdb_req_control_data_pull(uint8_t *buf, size_t buflen, ret = ctdb_uint32_pull(buf, buflen, mem_ctx, &cd->data.db_id); break; + + case CTDB_CONTROL_DB_PULL: + ret = ctdb_pulldb_ext_pull(buf, buflen, mem_ctx, + &cd->data.pulldb_ext); + break; + + case CTDB_CONTROL_DB_PUSH_START: + ret = ctdb_pulldb_ext_pull(buf, buflen, mem_ctx, + &cd->data.pulldb_ext); + break; + + case CTDB_CONTROL_DB_PUSH_CONFIRM: + ret = ctdb_uint32_pull(buf, buflen, mem_ctx, + &cd->data.db_id); + break; } return ret; @@ -1574,6 +1613,17 @@ static size_t ctdb_reply_control_data_len(struct ctdb_reply_control_data *cd) case CTDB_CONTROL_DB_TRANSACTION_CANCEL: break; + + case CTDB_CONTROL_DB_PULL: + len = ctdb_uint32_len(cd->data.num_records); + break; + + case CTDB_CONTROL_DB_PUSH_START: + break; + + case CTDB_CONTROL_DB_PUSH_CONFIRM: + len = ctdb_uint32_len(cd->data.num_records); + break; } return len; @@ -1726,6 +1776,14 @@ static void ctdb_reply_control_data_push(struct ctdb_reply_control_data *cd, case CTDB_CONTROL_GET_NODES_FILE: ctdb_node_map_push(cd->data.nodemap, buf); break; + + case CTDB_CONTROL_DB_PULL: + ctdb_uint32_push(cd->data.num_records, buf); + break; + + case CTDB_CONTROL_DB_PUSH_CONFIRM: + ctdb_uint32_push(cd->data.num_records, buf); + break; } } @@ -1913,6 +1971,16 @@ static int ctdb_reply_control_data_pull(uint8_t *buf, size_t buflen, ret = ctdb_node_map_pull(buf, buflen, mem_ctx, &cd->data.nodemap); break; + + case CTDB_CONTROL_DB_PULL: + ret = ctdb_uint32_pull(buf, buflen, mem_ctx, + &cd->data.num_records); + break; + + case CTDB_CONTROL_DB_PUSH_CONFIRM: + ret = ctdb_uint32_pull(buf, buflen, mem_ctx, + &cd->data.num_records); + break; } return ret; diff --git a/ctdb/tests/src/protocol_client_test.c b/ctdb/tests/src/protocol_client_test.c index f5a3c35..64923e9 100644 --- a/ctdb/tests/src/protocol_client_test.c +++ b/ctdb/tests/src/protocol_client_test.c @@ -656,6 +656,23 @@ static void fill_ctdb_req_control_data(TALLOC_CTX *mem_ctx, case CTDB_CONTROL_DB_TRANSACTION_CANCEL: cd->data.db_id = rand32(); break; + + case CTDB_CONTROL_DB_PULL: + cd->data.pulldb_ext = talloc(mem_ctx, struct ctdb_pulldb_ext); + assert(cd->data.pulldb_ext != NULL); + fill_ctdb_pulldb_ext(mem_ctx, cd->data.pulldb_ext); + break; + + case CTDB_CONTROL_DB_PUSH_START: + cd->data.pulldb_ext = talloc(mem_ctx, struct ctdb_pulldb_ext); + assert(cd->data.pulldb_ext != NULL); + fill_ctdb_pulldb_ext(mem_ctx, cd->data.pulldb_ext); + break; + + case CTDB_CONTROL_DB_PUSH_CONFIRM: + cd->data.db_id = rand32(); + break; + } } @@ -1103,6 +1120,21 @@ static void verify_ctdb_req_control_data(struct ctdb_req_control_data *cd, case CTDB_CONTROL_DB_TRANSACTION_CANCEL: assert(cd->data.db_id == cd2->data.db_id); break; + + case CTDB_CONTROL_DB_PULL: + verify_ctdb_pulldb_ext(cd->data.pulldb_ext, + cd2->data.pulldb_ext); + break; + + case CTDB_CONTROL_DB_PUSH_START: + verify_ctdb_pulldb_ext(cd->data.pulldb_ext, + cd2->data.pulldb_ext); + break; + + case CTDB_CONTROL_DB_PUSH_CONFIRM: + assert(cd->data.db_id == cd2->data.db_id); + break; + } } @@ -1559,6 +1591,14 @@ static void fill_ctdb_reply_control_data(TALLOC_CTX *mem_ctx, fill_ctdb_node_map(mem_ctx, cd->data.nodemap); break; + case CTDB_CONTROL_DB_PULL: + cd->data.num_records = rand32(); + break; + + case CTDB_CONTROL_DB_PUSH_CONFIRM: + cd->data.num_records = rand32(); + break; + } } @@ -1942,6 +1982,14 @@ static void verify_ctdb_reply_control_data(struct ctdb_reply_control_data *cd, verify_ctdb_node_map(cd->data.nodemap, cd2->data.nodemap); break; + case CTDB_CONTROL_DB_PULL: + assert(cd->data.num_records == cd2->data.num_records); + break; + + case CTDB_CONTROL_DB_PUSH_CONFIRM: + assert(cd->data.num_records == cd2->data.num_records); + break; + } } diff --git a/ctdb/tests/src/protocol_types_test.c b/ctdb/tests/src/protocol_types_test.c index bab104c..55ee743 100644 --- a/ctdb/tests/src/protocol_types_test.c +++ b/ctdb/tests/src/protocol_types_test.c @@ -182,6 +182,22 @@ static void verify_ctdb_pulldb(struct ctdb_pulldb *p1, struct ctdb_pulldb *p2) assert(p1->lmaster == p2->lmaster); } +static void fill_ctdb_pulldb_ext(TALLOC_CTX *mem_ctx, + struct ctdb_pulldb_ext *p) +{ + p->db_id = rand32(); + p->lmaster = rand32(); + p->srvid = rand64(); +} + +static void verify_ctdb_pulldb_ext(struct ctdb_pulldb_ext *p1, + struct ctdb_pulldb_ext *p2) +{ + assert(p1->db_id == p2->db_id); + assert(p1->lmaster == p2->lmaster); + assert(p1->srvid == p2->srvid); +} + static void fill_ctdb_ltdb_header(TALLOC_CTX *mem_ctx, struct ctdb_ltdb_header *p) { @@ -1177,6 +1193,7 @@ DEFINE_TEST(struct ctdb_statistics, ctdb_statistics); DEFINE_TEST(struct ctdb_vnn_map, ctdb_vnn_map); DEFINE_TEST(struct ctdb_dbid_map, ctdb_dbid_map); DEFINE_TEST(struct ctdb_pulldb, ctdb_pulldb); +DEFINE_TEST(struct ctdb_pulldb_ext, ctdb_pulldb_ext); DEFINE_TEST(struct ctdb_rec_data, ctdb_rec_data); DEFINE_TEST(struct ctdb_rec_buffer, ctdb_rec_buffer); DEFINE_TEST(struct ctdb_traverse_start, ctdb_traverse_start); @@ -1240,6 +1257,7 @@ int main(int argc, char *argv[]) TEST_FUNC(ctdb_vnn_map)(); TEST_FUNC(ctdb_dbid_map)(); TEST_FUNC(ctdb_pulldb)(); + TEST_FUNC(ctdb_pulldb_ext)(); TEST_FUNC(ctdb_rec_data)(); TEST_FUNC(ctdb_rec_buffer)(); TEST_FUNC(ctdb_traverse_start)(); -- 2.5.5 From 66191228517ab77a890b825531b05d6d350ce8e1 Mon Sep 17 00:00:00 2001 From: Amitay Isaacs Date: Fri, 19 Feb 2016 17:32:09 +1100 Subject: [PATCH 12/29] ctdb-daemon: Implement new controls DB_PULL and DB_PUSH_START/DB_PUSH_CONFIRM Signed-off-by: Amitay Isaacs Reviewed-by: Martin Schwenke (cherry picked from commit 95a15cde45c47e36d87d54464bcd769ee96e43c2) --- ctdb/include/ctdb_private.h | 11 ++ ctdb/server/ctdb_control.c | 12 ++ ctdb/server/ctdb_recover.c | 340 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 363 insertions(+) diff --git a/ctdb/include/ctdb_private.h b/ctdb/include/ctdb_private.h index b7c3e5d..04574fe 100644 --- a/ctdb/include/ctdb_private.h +++ b/ctdb/include/ctdb_private.h @@ -437,6 +437,9 @@ struct ctdb_db_context { bool freeze_transaction_started; uint32_t freeze_transaction_id; uint32_t generation; + + bool push_started; + void *push_state; }; @@ -873,6 +876,14 @@ int32_t ctdb_control_pull_db(struct ctdb_context *ctdb, TDB_DATA indata, TDB_DATA *outdata); int32_t ctdb_control_push_db(struct ctdb_context *ctdb, TDB_DATA indata); +int32_t ctdb_control_db_pull(struct ctdb_context *ctdb, + struct ctdb_req_control_old *c, + TDB_DATA indata, TDB_DATA *outdata); +int32_t ctdb_control_db_push_start(struct ctdb_context *ctdb, + TDB_DATA indata); +int32_t ctdb_control_db_push_confirm(struct ctdb_context *ctdb, + TDB_DATA indata, TDB_DATA *outdata); + int ctdb_deferred_drop_all_ips(struct ctdb_context *ctdb); int32_t ctdb_control_set_recmode(struct ctdb_context *ctdb, diff --git a/ctdb/server/ctdb_control.c b/ctdb/server/ctdb_control.c index e6f8a0d..d840174 100644 --- a/ctdb/server/ctdb_control.c +++ b/ctdb/server/ctdb_control.c @@ -715,6 +715,18 @@ static int32_t ctdb_control_dispatch(struct ctdb_context *ctdb, CHECK_CONTROL_DATA_SIZE(sizeof(uint32_t)); return ctdb_control_db_transaction_cancel(ctdb, indata); + case CTDB_CONTROL_DB_PULL: + CHECK_CONTROL_DATA_SIZE(sizeof(struct ctdb_pulldb_ext)); + return ctdb_control_db_pull(ctdb, c, indata, outdata); + + case CTDB_CONTROL_DB_PUSH_START: + CHECK_CONTROL_DATA_SIZE(sizeof(struct ctdb_pulldb_ext)); + return ctdb_control_db_push_start(ctdb, indata); + + case CTDB_CONTROL_DB_PUSH_CONFIRM: + CHECK_CONTROL_DATA_SIZE(sizeof(uint32_t)); + return ctdb_control_db_push_confirm(ctdb, indata, outdata); + default: DEBUG(DEBUG_CRIT,(__location__ " Unknown CTDB control opcode %u\n", opcode)); return -1; diff --git a/ctdb/server/ctdb_recover.c b/ctdb/server/ctdb_recover.c index 127ff6b..49f568a 100644 --- a/ctdb/server/ctdb_recover.c +++ b/ctdb/server/ctdb_recover.c @@ -313,6 +313,133 @@ int32_t ctdb_control_pull_db(struct ctdb_context *ctdb, TDB_DATA indata, TDB_DAT return 0; } +struct db_pull_state { + struct ctdb_context *ctdb; + struct ctdb_db_context *ctdb_db; + struct ctdb_marshall_buffer *recs; + uint32_t pnn; + uint64_t srvid; + uint32_t num_records; +}; + +static int traverse_db_pull(struct tdb_context *tdb, TDB_DATA key, + TDB_DATA data, void *private_data) +{ + struct db_pull_state *state = (struct db_pull_state *)private_data; + struct ctdb_marshall_buffer *recs; + + recs = ctdb_marshall_add(state->ctdb, state->recs, + state->ctdb_db->db_id, 0, key, NULL, data); + if (recs == NULL) { + TALLOC_FREE(state->recs); + return -1; + } + state->recs = recs; + + if (talloc_get_size(state->recs) >= + state->ctdb->tunable.rec_buffer_size_limit) { + TDB_DATA buffer; + int ret; + + buffer = ctdb_marshall_finish(state->recs); + ret = ctdb_daemon_send_message(state->ctdb, state->pnn, + state->srvid, buffer); + if (ret != 0) { + TALLOC_FREE(state->recs); + return -1; + } + + state->num_records += state->recs->count; + TALLOC_FREE(state->recs); + } + + return 0; +} + +int32_t ctdb_control_db_pull(struct ctdb_context *ctdb, + struct ctdb_req_control_old *c, + TDB_DATA indata, TDB_DATA *outdata) +{ + struct ctdb_pulldb_ext *pulldb_ext; + struct ctdb_db_context *ctdb_db; + struct db_pull_state state; + int ret; + + pulldb_ext = (struct ctdb_pulldb_ext *)indata.dptr; + + ctdb_db = find_ctdb_db(ctdb, pulldb_ext->db_id); + if (ctdb_db == NULL) { + DEBUG(DEBUG_ERR,(__location__ " Unknown db 0x%08x\n", + pulldb_ext->db_id)); + return -1; + } + + if (!ctdb_db_frozen(ctdb_db)) { + DEBUG(DEBUG_ERR, + ("rejecting ctdb_control_pull_db when not frozen\n")); + return -1; + } + + if (ctdb_db->unhealthy_reason) { + /* this is just a warning, as the tdb should be empty anyway */ + DEBUG(DEBUG_WARNING, + ("db(%s) unhealty in ctdb_control_db_pull: %s\n", + ctdb_db->db_name, ctdb_db->unhealthy_reason)); + } + + state.ctdb = ctdb; + state.ctdb_db = ctdb_db; + state.recs = NULL; + state.pnn = c->hdr.srcnode; + state.srvid = pulldb_ext->srvid; + state.num_records = 0; + + if (ctdb_lockdb_mark(ctdb_db) != 0) { + DEBUG(DEBUG_ERR, + (__location__ " Failed to get lock on entire db - failing\n")); + return -1; + } + + ret = tdb_traverse_read(ctdb_db->ltdb->tdb, traverse_db_pull, &state); + if (ret == -1) { + DEBUG(DEBUG_ERR, + (__location__ " Failed to get traverse db '%s'\n", + ctdb_db->db_name)); + ctdb_lockdb_unmark(ctdb_db); + return -1; + } + + /* Last few records */ + if (state.recs != NULL) { + TDB_DATA buffer; + + buffer = ctdb_marshall_finish(state.recs); + ret = ctdb_daemon_send_message(state.ctdb, state.pnn, + state.srvid, buffer); + if (ret != 0) { + TALLOC_FREE(state.recs); + ctdb_lockdb_unmark(ctdb_db); + return -1; + } + + state.num_records += state.recs->count; + TALLOC_FREE(state.recs); + } + + ctdb_lockdb_unmark(ctdb_db); + + outdata->dptr = talloc_size(outdata, sizeof(uint32_t)); + if (outdata->dptr == NULL) { + DEBUG(DEBUG_ERR, (__location__ " Memory allocation error\n")); + return -1; + } + + memcpy(outdata->dptr, (uint8_t *)&state.num_records, sizeof(uint32_t)); + outdata->dsize = sizeof(uint32_t); + + return 0; +} + /* push a bunch of records into a ltdb, filtering by rsn */ @@ -407,6 +534,219 @@ failed: return -1; } +struct db_push_state { + struct ctdb_context *ctdb; + struct ctdb_db_context *ctdb_db; + uint64_t srvid; + uint32_t num_records; + bool failed; +}; + +static void db_push_msg_handler(uint64_t srvid, TDB_DATA indata, + void *private_data) +{ + struct db_push_state *state = talloc_get_type( + private_data, struct db_push_state); + struct ctdb_marshall_buffer *recs; + struct ctdb_rec_data_old *rec; + int i, ret; + + if (state->failed) { + return; + } + + recs = (struct ctdb_marshall_buffer *)indata.dptr; + rec = (struct ctdb_rec_data_old *)&recs->data[0]; + + DEBUG(DEBUG_INFO, ("starting push of %u records for dbid 0x%x\n", + recs->count, recs->db_id)); + + for (i=0; icount; i++) { + TDB_DATA key, data; + struct ctdb_ltdb_header *hdr; + + key.dptr = &rec->data[0]; + key.dsize = rec->keylen; + data.dptr = &rec->data[key.dsize]; + data.dsize = rec->datalen; + + if (data.dsize < sizeof(struct ctdb_ltdb_header)) { + DEBUG(DEBUG_CRIT,(__location__ " bad ltdb record\n")); + goto failed; + } + + hdr = (struct ctdb_ltdb_header *)data.dptr; + /* Strip off any read only record flags. + * All readonly records are revoked implicitely by a recovery. + */ + hdr->flags &= ~CTDB_REC_RO_FLAGS; + + data.dptr += sizeof(*hdr); + data.dsize -= sizeof(*hdr); + + ret = ctdb_ltdb_store(state->ctdb_db, key, hdr, data); + if (ret != 0) { + DEBUG(DEBUG_ERR, + (__location__ " Unable to store record\n")); + goto failed; + } + + rec = (struct ctdb_rec_data_old *)(rec->length + (uint8_t *)rec); + } + + DEBUG(DEBUG_DEBUG, ("finished push of %u records for dbid 0x%x\n", + recs->count, recs->db_id)); + + state->num_records += recs->count; + return; + +failed: + state->failed = true; +} + +int32_t ctdb_control_db_push_start(struct ctdb_context *ctdb, TDB_DATA indata) +{ + struct ctdb_pulldb_ext *pulldb_ext; + struct ctdb_db_context *ctdb_db; + struct db_push_state *state; + int ret; + + pulldb_ext = (struct ctdb_pulldb_ext *)indata.dptr; + + ctdb_db = find_ctdb_db(ctdb, pulldb_ext->db_id); + if (ctdb_db == NULL) { + DEBUG(DEBUG_ERR, + (__location__ " Unknown db 0x%08x\n", pulldb_ext->db_id)); + return -1; + } + + if (!ctdb_db_frozen(ctdb_db)) { + DEBUG(DEBUG_ERR, + ("rejecting ctdb_control_db_push_start when not frozen\n")); + return -1; + } + + if (ctdb_db->push_started) { + DEBUG(DEBUG_WARNING, + (__location__ " DB push already started for %s\n", + ctdb_db->db_name)); + + /* De-register old state */ + state = (struct db_push_state *)ctdb_db->push_state; + if (state != NULL) { + srvid_deregister(ctdb->srv, state->srvid, state); + talloc_free(state); + ctdb_db->push_state = NULL; + } + } + + state = talloc_zero(ctdb_db, struct db_push_state); + if (state == NULL) { + DEBUG(DEBUG_ERR, (__location__ " Memory allocation error\n")); + return -1; + } + + state->ctdb = ctdb; + state->ctdb_db = ctdb_db; + state->srvid = pulldb_ext->srvid; + state->failed = false; + + ret = srvid_register(ctdb->srv, state, state->srvid, + db_push_msg_handler, state); + if (ret != 0) { + DEBUG(DEBUG_ERR, + (__location__ " Failed to register srvid for db push\n")); + talloc_free(state); + return -1; + } + + if (ctdb_lockdb_mark(ctdb_db) != 0) { + DEBUG(DEBUG_ERR, + (__location__ " Failed to get lock on entire db - failing\n")); + srvid_deregister(ctdb->srv, state->srvid, state); + talloc_free(state); + return -1; + } + + ctdb_db->push_started = true; + ctdb_db->push_state = state; + + return 0; +} + +int32_t ctdb_control_db_push_confirm(struct ctdb_context *ctdb, + TDB_DATA indata, TDB_DATA *outdata) +{ + uint32_t db_id; + struct ctdb_db_context *ctdb_db; + struct db_push_state *state; + + db_id = *(uint32_t *)indata.dptr; + + ctdb_db = find_ctdb_db(ctdb, db_id); + if (ctdb_db == NULL) { + DEBUG(DEBUG_ERR,(__location__ " Unknown db 0x%08x\n", db_id)); + return -1; + } + + if (!ctdb_db_frozen(ctdb_db)) { + DEBUG(DEBUG_ERR, + ("rejecting ctdb_control_db_push_confirm when not frozen\n")); + return -1; + } + + if (!ctdb_db->push_started) { + DEBUG(DEBUG_ERR, (__location__ " DB push not started\n")); + return -1; + } + + if (ctdb_db->readonly) { + DEBUG(DEBUG_ERR, + ("Clearing the tracking database for dbid 0x%x\n", + ctdb_db->db_id)); + if (tdb_wipe_all(ctdb_db->rottdb) != 0) { + DEBUG(DEBUG_ERR, + ("Failed to wipe tracking database for 0x%x." + " Dropping read-only delegation support\n", + ctdb_db->db_id)); + ctdb_db->readonly = false; + tdb_close(ctdb_db->rottdb); + ctdb_db->rottdb = NULL; + ctdb_db->readonly = false; + } + + while (ctdb_db->revokechild_active != NULL) { + talloc_free(ctdb_db->revokechild_active); + } + } + + ctdb_lockdb_unmark(ctdb_db); + + state = (struct db_push_state *)ctdb_db->push_state; + if (state == NULL) { + DEBUG(DEBUG_ERR, (__location__ " Missing push db state\n")); + return -1; + } + + srvid_deregister(ctdb->srv, state->srvid, state); + + outdata->dptr = talloc_size(outdata, sizeof(uint32_t)); + if (outdata->dptr == NULL) { + DEBUG(DEBUG_ERR, (__location__ " Memory allocation error\n")); + talloc_free(state); + ctdb_db->push_state = NULL; + return -1; + } + + memcpy(outdata->dptr, (uint8_t *)&state->num_records, sizeof(uint32_t)); + outdata->dsize = sizeof(uint32_t); + + talloc_free(state); + ctdb_db->push_state = NULL; + + return 0; +} + struct ctdb_set_recmode_state { struct ctdb_context *ctdb; struct ctdb_req_control_old *c; -- 2.5.5 From cab299abb4029cfe7bf6aac741751e3404ea63d7 Mon Sep 17 00:00:00 2001 From: Amitay Isaacs Date: Fri, 19 Feb 2016 11:14:40 +1100 Subject: [PATCH 13/29] ctdb-client: Add client API functions for new controls Signed-off-by: Amitay Isaacs Reviewed-by: Martin Schwenke (cherry picked from commit 338e0dccd903b63bf15ae9af38c47aad01c110d0) --- ctdb/client/client.h | 15 +++++++ ctdb/client/client_control_sync.c | 86 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 101 insertions(+) diff --git a/ctdb/client/client.h b/ctdb/client/client.h index 66be313..eebb7a6 100644 --- a/ctdb/client/client.h +++ b/ctdb/client/client.h @@ -737,6 +737,21 @@ int ctdb_ctrl_db_transaction_cancel(TALLOC_CTX *mem_ctx, int destnode, struct timeval timeout, uint32_t db_id); +int ctdb_ctrl_db_pull(TALLOC_CTX *mem_ctx, struct tevent_context *ev, + struct ctdb_client_context *client, + int destnode, struct timeval timeout, + struct ctdb_pulldb_ext *pulldb, uint32_t *num_records); + +int ctdb_ctrl_db_push_start(TALLOC_CTX *mem_ctx, struct tevent_context *ev, + struct ctdb_client_context *client, + int destnode, struct timeval timeout, + struct ctdb_pulldb_ext *pulldb); + +int ctdb_ctrl_db_push_confirm(TALLOC_CTX *mem_ctx, struct tevent_context *ev, + struct ctdb_client_context *client, + int destnode, struct timeval timeout, + uint32_t db_id, uint32_t *num_records); + /* from client/client_db.c */ struct tevent_req *ctdb_attach_send(TALLOC_CTX *mem_ctx, diff --git a/ctdb/client/client_control_sync.c b/ctdb/client/client_control_sync.c index de52b47..753007e 100644 --- a/ctdb/client/client_control_sync.c +++ b/ctdb/client/client_control_sync.c @@ -3117,3 +3117,89 @@ int ctdb_ctrl_db_transaction_cancel(TALLOC_CTX *mem_ctx, return 0; } + +int ctdb_ctrl_db_pull(TALLOC_CTX *mem_ctx, struct tevent_context *ev, + struct ctdb_client_context *client, + int destnode, struct timeval timeout, + struct ctdb_pulldb_ext *pulldb, uint32_t *num_records) +{ + struct ctdb_req_control request; + struct ctdb_reply_control *reply; + int ret; + + ctdb_req_control_db_pull(&request, pulldb); + ret = ctdb_client_control(mem_ctx, ev, client, destnode, timeout, + &request, &reply); + if (ret != 0) { + DEBUG(DEBUG_ERR, + ("Control DB_PULL failed to node %u, ret=%d\n", + destnode, ret)); + return ret; + } + + ret = ctdb_reply_control_db_pull(reply, num_records); + if (ret != 0) { + DEBUG(DEBUG_ERR, ("Control DB_PULL failed, ret=%d\n", ret)); + return ret; + } + + return 0; +} + +int ctdb_ctrl_db_push_start(TALLOC_CTX *mem_ctx, struct tevent_context *ev, + struct ctdb_client_context *client, + int destnode, struct timeval timeout, + struct ctdb_pulldb_ext *pulldb) +{ + struct ctdb_req_control request; + struct ctdb_reply_control *reply; + int ret; + + ctdb_req_control_db_push_start(&request, pulldb); + ret = ctdb_client_control(mem_ctx, ev, client, destnode, timeout, + &request, &reply); + if (ret != 0) { + DEBUG(DEBUG_ERR, + ("Control DB_PUSH failed to node %u, ret=%d\n", + destnode, ret)); + return ret; + } + + ret = ctdb_reply_control_db_push_start(reply); + if (ret != 0) { + DEBUG(DEBUG_ERR, + ("Control DB_PUSH failed, ret=%d\n", ret)); + return ret; + } + + return 0; +} + +int ctdb_ctrl_db_push_confirm(TALLOC_CTX *mem_ctx, struct tevent_context *ev, + struct ctdb_client_context *client, + int destnode, struct timeval timeout, + uint32_t db_id, uint32_t *num_records) +{ + struct ctdb_req_control request; + struct ctdb_reply_control *reply; + int ret; + + ctdb_req_control_db_push_confirm(&request, db_id); + ret = ctdb_client_control(mem_ctx, ev, client, destnode, timeout, + &request, &reply); + if (ret != 0) { + DEBUG(DEBUG_ERR, + ("Control DB_PUSH failed to node %u, ret=%d\n", + destnode, ret)); + return ret; + } + + ret = ctdb_reply_control_db_push_confirm(reply, num_records); + if (ret != 0) { + DEBUG(DEBUG_ERR, + ("Control DB_PUSH failed, ret=%d\n", ret)); + return ret; + } + + return 0; +} -- 2.5.5 From c1a2d1001a296e5d02ec864fe25f2417e4409bdb Mon Sep 17 00:00:00 2001 From: Amitay Isaacs Date: Fri, 26 Feb 2016 17:36:39 +1100 Subject: [PATCH 14/29] ctdb-recovery-helper: Factor out generic recv function Signed-off-by: Amitay Isaacs Reviewed-by: Martin Schwenke (cherry picked from commit 5f43f92796f4f8130067e23555f94842bb803fcf) --- ctdb/server/ctdb_recovery_helper.c | 53 +++++++++++++------------------------- 1 file changed, 18 insertions(+), 35 deletions(-) diff --git a/ctdb/server/ctdb_recovery_helper.c b/ctdb/server/ctdb_recovery_helper.c index 607faa6..4338818 100644 --- a/ctdb/server/ctdb_recovery_helper.c +++ b/ctdb/server/ctdb_recovery_helper.c @@ -65,6 +65,20 @@ static ssize_t sys_write(int fd, const void *buf, size_t count) return ret; } +static bool generic_recv(struct tevent_req *req, int *perr) +{ + int err; + + if (tevent_req_is_unix_error(req, &err)) { + if (perr != NULL) { + *perr = err; + } + return false; + } + + return true; +} + /* * Recovery database functions */ @@ -443,16 +457,7 @@ static void collect_highseqnum_db_pulldb_done(struct tevent_req *subreq) static bool collect_highseqnum_db_recv(struct tevent_req *req, int *perr) { - int err; - - if (tevent_req_is_unix_error(req, &err)) { - if (perr != NULL) { - *perr = err; - } - return false; - } - - return true; + return generic_recv(req, perr); } /* @@ -570,16 +575,7 @@ static void collect_all_db_pulldb_done(struct tevent_req *subreq) static bool collect_all_db_recv(struct tevent_req *req, int *perr) { - int err; - - if (tevent_req_is_unix_error(req, &err)) { - if (perr != NULL) { - *perr = err; - } - return false; - } - - return true; + return generic_recv(req, perr); } @@ -1050,13 +1046,7 @@ static void recover_db_thaw_done(struct tevent_req *subreq) static bool recover_db_recv(struct tevent_req *req) { - int err; - - if (tevent_req_is_unix_error(req, &err)) { - return false; - } - - return true; + return generic_recv(req, NULL); } @@ -1808,14 +1798,7 @@ static void recovery_end_recovery_done(struct tevent_req *subreq) static void recovery_recv(struct tevent_req *req, int *perr) { - int err; - - if (tevent_req_is_unix_error(req, &err)) { - if (perr != NULL) { - *perr = err; - } - return; - } + generic_recv(req, perr); } static void usage(const char *progname) -- 2.5.5 From 5c8d980997d48c25edff07240cc6d5a552e46ebe Mon Sep 17 00:00:00 2001 From: Amitay Isaacs Date: Thu, 25 Feb 2016 11:04:51 +1100 Subject: [PATCH 15/29] ctdb-recovery-helper: Pass capabilities to database recovery functions Signed-off-by: Amitay Isaacs Reviewed-by: Martin Schwenke (cherry picked from commit 5b926d882e3cf63d4ad3a9714fd0d440d6b5f3e5) --- ctdb/server/ctdb_recovery_helper.c | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/ctdb/server/ctdb_recovery_helper.c b/ctdb/server/ctdb_recovery_helper.c index 4338818..ef5303f 100644 --- a/ctdb/server/ctdb_recovery_helper.c +++ b/ctdb/server/ctdb_recovery_helper.c @@ -304,6 +304,7 @@ struct collect_highseqnum_db_state { struct ctdb_client_context *client; uint32_t *pnn_list; int count; + uint32_t *caps; uint32_t db_id; struct recdb_context *recdb; uint32_t max_pnn; @@ -316,7 +317,7 @@ static struct tevent_req *collect_highseqnum_db_send( TALLOC_CTX *mem_ctx, struct tevent_context *ev, struct ctdb_client_context *client, - uint32_t *pnn_list, int count, + uint32_t *pnn_list, int count, uint32_t *caps, uint32_t db_id, struct recdb_context *recdb) { struct tevent_req *req, *subreq; @@ -333,6 +334,7 @@ static struct tevent_req *collect_highseqnum_db_send( state->client = client; state->pnn_list = pnn_list; state->count = count; + state->caps = caps; state->db_id = db_id; state->recdb = recdb; @@ -469,6 +471,7 @@ struct collect_all_db_state { struct ctdb_client_context *client; uint32_t *pnn_list; int count; + uint32_t *caps; uint32_t db_id; struct recdb_context *recdb; struct ctdb_pulldb pulldb; @@ -481,7 +484,7 @@ static struct tevent_req *collect_all_db_send( TALLOC_CTX *mem_ctx, struct tevent_context *ev, struct ctdb_client_context *client, - uint32_t *pnn_list, int count, + uint32_t *pnn_list, int count, uint32_t *caps, uint32_t db_id, struct recdb_context *recdb) { struct tevent_req *req, *subreq; @@ -598,6 +601,7 @@ struct recover_db_state { struct ctdb_tunable_list *tun_list; uint32_t *pnn_list; int count; + uint32_t *caps; uint32_t db_id; bool persistent; @@ -625,6 +629,7 @@ static struct tevent_req *recover_db_send(TALLOC_CTX *mem_ctx, struct ctdb_client_context *client, struct ctdb_tunable_list *tun_list, uint32_t *pnn_list, int count, + uint32_t *caps, uint32_t generation, uint32_t db_id, bool persistent) { @@ -642,6 +647,7 @@ static struct tevent_req *recover_db_send(TALLOC_CTX *mem_ctx, state->tun_list = tun_list; state->pnn_list = pnn_list; state->count = count; + state->caps = caps; state->db_id = db_id; state->persistent = persistent; @@ -826,12 +832,12 @@ static void recover_db_transaction_started(struct tevent_req *subreq) if (state->persistent && state->tun_list->recover_pdb_by_seqnum != 0) { subreq = collect_highseqnum_db_send( state, state->ev, state->client, - state->pnn_list, state->count, + state->pnn_list, state->count, state->caps, state->db_id, state->recdb); } else { subreq = collect_all_db_send( state, state->ev, state->client, - state->pnn_list, state->count, + state->pnn_list, state->count, state->caps, state->db_id, state->recdb); } if (tevent_req_nomem(subreq, req)) { @@ -1070,6 +1076,7 @@ struct db_recovery_one_state { struct ctdb_tunable_list *tun_list; uint32_t *pnn_list; int count; + uint32_t *caps; uint32_t generation; uint32_t db_id; bool persistent; @@ -1084,6 +1091,7 @@ static struct tevent_req *db_recovery_send(TALLOC_CTX *mem_ctx, struct ctdb_dbid_map *dbmap, struct ctdb_tunable_list *tun_list, uint32_t *pnn_list, int count, + uint32_t *caps, uint32_t generation) { struct tevent_req *req, *subreq; @@ -1119,14 +1127,15 @@ static struct tevent_req *db_recovery_send(TALLOC_CTX *mem_ctx, substate->tun_list = tun_list; substate->pnn_list = pnn_list; substate->count = count; + substate->caps = caps; substate->generation = generation; substate->db_id = dbmap->dbs[i].db_id; substate->persistent = dbmap->dbs[i].flags & CTDB_DB_FLAGS_PERSISTENT; subreq = recover_db_send(state, ev, client, tun_list, - pnn_list, count, generation, - substate->db_id, + pnn_list, count, caps, + generation, substate->db_id, substate->persistent); if (tevent_req_nomem(subreq, req)) { return tevent_req_post(req, ev); @@ -1161,6 +1170,7 @@ static void db_recovery_one_done(struct tevent_req *subreq) subreq = recover_db_send(state, state->ev, substate->client, substate->tun_list, substate->pnn_list, substate->count, + substate->caps, substate->generation, substate->db_id, substate->persistent); if (tevent_req_nomem(subreq, req)) { @@ -1678,7 +1688,7 @@ static void recovery_vnnmap_update_done(struct tevent_req *subreq) subreq = db_recovery_send(state, state->ev, state->client, state->dbmap, state->tun_list, state->pnn_list, state->count, - state->vnnmap->generation); + state->caps, state->vnnmap->generation); if (tevent_req_nomem(subreq, req)) { return; } -- 2.5.5 From 097623158c286f4255f3f4575e032d7b509014f7 Mon Sep 17 00:00:00 2001 From: Amitay Isaacs Date: Fri, 26 Feb 2016 15:42:53 +1100 Subject: [PATCH 16/29] ctdb-recovery-helper: Rename pnn to dmaster in recdb_records() This variable is used to set the dmaster value for each record in recdb_traverse(). Signed-off-by: Amitay Isaacs Reviewed-by: Martin Schwenke (cherry picked from commit 70011a1bfb24d9f4b2a042d353f907ef7c49bc1c) --- ctdb/server/ctdb_recovery_helper.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/ctdb/server/ctdb_recovery_helper.c b/ctdb/server/ctdb_recovery_helper.c index ef5303f..de9c3da 100644 --- a/ctdb/server/ctdb_recovery_helper.c +++ b/ctdb/server/ctdb_recovery_helper.c @@ -203,7 +203,7 @@ static bool recdb_add(struct recdb_context *recdb, int mypnn, struct recdb_traverse_state { struct ctdb_rec_buffer *recbuf; - uint32_t pnn; + uint32_t dmaster; uint32_t reqid; bool persistent; bool failed; @@ -255,7 +255,7 @@ static int recdb_traverse(struct tdb_context *tdb, TDB_DATA key, TDB_DATA data, /* update the dmaster field to point to us */ header = (struct ctdb_ltdb_header *)data.dptr; if (!state->persistent) { - header->dmaster = state->pnn; + header->dmaster = state->dmaster; header->flags |= CTDB_REC_FLAG_MIGRATED_WITH_DATA; } @@ -270,7 +270,8 @@ static int recdb_traverse(struct tdb_context *tdb, TDB_DATA key, TDB_DATA data, } static struct ctdb_rec_buffer *recdb_records(struct recdb_context *recdb, - TALLOC_CTX *mem_ctx, uint32_t pnn) + TALLOC_CTX *mem_ctx, + uint32_t dmaster) { struct recdb_traverse_state state; int ret; @@ -279,7 +280,7 @@ static struct ctdb_rec_buffer *recdb_records(struct recdb_context *recdb, if (state.recbuf == NULL) { return NULL; } - state.pnn = pnn; + state.dmaster = dmaster; state.reqid = 0; state.persistent = recdb->persistent; state.failed = false; -- 2.5.5 From 6e208aca8683b5157fc99053285143e2ecf39277 Mon Sep 17 00:00:00 2001 From: Amitay Isaacs Date: Mon, 29 Feb 2016 13:53:52 +1100 Subject: [PATCH 17/29] ctdb-recovery-helper: Create accessors for recdb structure fields Signed-off-by: Amitay Isaacs Reviewed-by: Martin Schwenke (cherry picked from commit a80ff09ed3da49c544fb121ae88e3ccb351fc4e7) --- ctdb/server/ctdb_recovery_helper.c | 32 ++++++++++++++++++++++++++------ 1 file changed, 26 insertions(+), 6 deletions(-) diff --git a/ctdb/server/ctdb_recovery_helper.c b/ctdb/server/ctdb_recovery_helper.c index de9c3da..d7a2b5b 100644 --- a/ctdb/server/ctdb_recovery_helper.c +++ b/ctdb/server/ctdb_recovery_helper.c @@ -135,11 +135,31 @@ static struct recdb_context *recdb_create(TALLOC_CTX *mem_ctx, uint32_t db_id, return recdb; } +static uint32_t recdb_id(struct recdb_context *recdb) +{ + return recdb->db_id; +} + static const char *recdb_name(struct recdb_context *recdb) { return recdb->db_name; } +static const char *recdb_path(struct recdb_context *recdb) +{ + return recdb->db_path; +} + +static struct tdb_context *recdb_tdb(struct recdb_context *recdb) +{ + return recdb->db->tdb; +} + +static bool recdb_persistent(struct recdb_context *recdb) +{ + return recdb->persistent; +} + struct recdb_add_traverse_state { struct recdb_context *recdb; int mypnn; @@ -163,7 +183,7 @@ static int recdb_add_traverse(uint32_t reqid, struct ctdb_ltdb_header *header, hdr = (struct ctdb_ltdb_header *)data.dptr; /* fetch the existing record, if any */ - prev_data = tdb_fetch(state->recdb->db->tdb, key); + prev_data = tdb_fetch(recdb_tdb(state->recdb), key); if (prev_data.dptr != NULL) { struct ctdb_ltdb_header prev_hdr; @@ -177,7 +197,7 @@ static int recdb_add_traverse(uint32_t reqid, struct ctdb_ltdb_header *header, } } - ret = tdb_store(state->recdb->db->tdb, key, data, TDB_REPLACE); + ret = tdb_store(recdb_tdb(state->recdb), key, data, TDB_REPLACE); if (ret != 0) { return -1; } @@ -276,19 +296,19 @@ static struct ctdb_rec_buffer *recdb_records(struct recdb_context *recdb, struct recdb_traverse_state state; int ret; - state.recbuf = ctdb_rec_buffer_init(mem_ctx, recdb->db_id); + state.recbuf = ctdb_rec_buffer_init(mem_ctx, recdb_id(recdb)); if (state.recbuf == NULL) { return NULL; } state.dmaster = dmaster; state.reqid = 0; - state.persistent = recdb->persistent; + state.persistent = recdb_persistent(recdb); state.failed = false; - ret = tdb_traverse_read(recdb->db->tdb, recdb_traverse, &state); + ret = tdb_traverse_read(recdb_tdb(recdb), recdb_traverse, &state); if (ret == -1 || state.failed) { LOG("Failed to marshall recovery records for %s\n", - recdb->db_name); + recdb_name(recdb)); TALLOC_FREE(state.recbuf); return NULL; } -- 2.5.5 From e332108f0a4f6c523121a473d303a3bee4916292 Mon Sep 17 00:00:00 2001 From: Amitay Isaacs Date: Tue, 8 Mar 2016 17:20:30 +1100 Subject: [PATCH 18/29] ctdb-protocol: Add file IO functions for ctdb_rec_buffer Signed-off-by: Amitay Isaacs Reviewed-by: Martin Schwenke (cherry picked from commit 76f653f0bc016dfa98a4d8688469be4ac450a17a) --- ctdb/protocol/protocol_api.h | 4 +++ ctdb/protocol/protocol_types.c | 62 ++++++++++++++++++++++++++++++++++++ ctdb/tests/src/protocol_types_test.c | 49 ++++++++++++++++++++++++++++ 3 files changed, 115 insertions(+) diff --git a/ctdb/protocol/protocol_api.h b/ctdb/protocol/protocol_api.h index ed7bdcb..4cd8b11 100644 --- a/ctdb/protocol/protocol_api.h +++ b/ctdb/protocol/protocol_api.h @@ -52,6 +52,10 @@ int ctdb_rec_buffer_traverse(struct ctdb_rec_buffer *recbuf, ctdb_rec_parser_func_t func, void *private_data); +int ctdb_rec_buffer_write(struct ctdb_rec_buffer *recbuf, int fd); +int ctdb_rec_buffer_read(int fd, TALLOC_CTX *mem_ctx, + struct ctdb_rec_buffer **out); + size_t ctdb_server_id_len(struct ctdb_server_id *sid); void ctdb_server_id_push(struct ctdb_server_id *sid, uint8_t *buf); int ctdb_server_id_pull(uint8_t *buf, size_t buflen, diff --git a/ctdb/protocol/protocol_types.c b/ctdb/protocol/protocol_types.c index a026333..cd04aca 100644 --- a/ctdb/protocol/protocol_types.c +++ b/ctdb/protocol/protocol_types.c @@ -800,6 +800,68 @@ int ctdb_rec_buffer_traverse(struct ctdb_rec_buffer *recbuf, return ret; } +int ctdb_rec_buffer_write(struct ctdb_rec_buffer *recbuf, int fd) +{ + ssize_t n; + + n = write(fd, &recbuf->db_id, sizeof(uint32_t)); + if (n == -1 || n != sizeof(uint32_t)) { + return (errno != 0 ? errno : EIO); + } + n = write(fd, &recbuf->count, sizeof(uint32_t)); + if (n == -1 || n != sizeof(uint32_t)) { + return (errno != 0 ? errno : EIO); + } + n = write(fd, &recbuf->buflen, sizeof(size_t)); + if (n == -1 || n != sizeof(size_t)) { + return (errno != 0 ? errno : EIO); + } + n = write(fd, recbuf->buf, recbuf->buflen); + if (n == -1 || n != recbuf->buflen) { + return (errno != 0 ? errno : EIO); + } + + return 0; +} + +int ctdb_rec_buffer_read(int fd, TALLOC_CTX *mem_ctx, + struct ctdb_rec_buffer **out) +{ + struct ctdb_rec_buffer *recbuf; + ssize_t n; + + recbuf = talloc(mem_ctx, struct ctdb_rec_buffer); + if (recbuf == NULL) { + return ENOMEM; + } + + n = read(fd, &recbuf->db_id, sizeof(uint32_t)); + if (n == -1 || n != sizeof(uint32_t)) { + return (errno != 0 ? errno : EIO); + } + n = read(fd, &recbuf->count, sizeof(uint32_t)); + if (n == -1 || n != sizeof(uint32_t)) { + return (errno != 0 ? errno : EIO); + } + n = read(fd, &recbuf->buflen, sizeof(size_t)); + if (n == -1 || n != sizeof(size_t)) { + return (errno != 0 ? errno : EIO); + } + + recbuf->buf = talloc_size(recbuf, recbuf->buflen); + if (recbuf->buf == NULL) { + return ENOMEM; + } + + n = read(fd, recbuf->buf, recbuf->buflen); + if (n == -1 || n != recbuf->buflen) { + return (errno != 0 ? errno : EIO); + } + + *out = recbuf; + return 0; +} + size_t ctdb_traverse_start_len(struct ctdb_traverse_start *traverse) { return sizeof(struct ctdb_traverse_start); diff --git a/ctdb/tests/src/protocol_types_test.c b/ctdb/tests/src/protocol_types_test.c index 55ee743..e39a2c9 100644 --- a/ctdb/tests/src/protocol_types_test.c +++ b/ctdb/tests/src/protocol_types_test.c @@ -19,6 +19,7 @@ #include "replace.h" #include "system/network.h" +#include "system/filesys.h" #include @@ -1235,6 +1236,52 @@ DEFINE_TEST(struct ctdb_srvid_message, ctdb_srvid_message); DEFINE_TEST(struct ctdb_disable_message, ctdb_disable_message); DEFINE_TEST(struct ctdb_g_lock_list, ctdb_g_lock_list); +static void test_ctdb_rec_buffer_read_write(void) +{ + TALLOC_CTX *mem_ctx = talloc_new(NULL); + struct ctdb_rec_buffer *p1, **p2; + const char *filename = "ctdb_rec_buffer_test.dat"; + int count = 100; + int fd, i, ret; + off_t offset; + + p1 = talloc_array(mem_ctx, struct ctdb_rec_buffer, count); + assert(p1 != NULL); + for (i=0; i Date: Tue, 8 Mar 2016 17:48:46 +1100 Subject: [PATCH 19/29] ctdb-recovery-helper: Re-factor function to retain records from recdb Also, rename traverse function and traverse state for recdb_records consistently. Signed-off-by: Amitay Isaacs Reviewed-by: Martin Schwenke (cherry picked from commit 9058fe06df7f24e9a1b8fd5792a537a1fa8f5a60) --- ctdb/server/ctdb_recovery_helper.c | 56 ++++++++++++++++++++++++-------------- 1 file changed, 36 insertions(+), 20 deletions(-) diff --git a/ctdb/server/ctdb_recovery_helper.c b/ctdb/server/ctdb_recovery_helper.c index d7a2b5b..05900ef 100644 --- a/ctdb/server/ctdb_recovery_helper.c +++ b/ctdb/server/ctdb_recovery_helper.c @@ -221,19 +221,11 @@ static bool recdb_add(struct recdb_context *recdb, int mypnn, return true; } -struct recdb_traverse_state { - struct ctdb_rec_buffer *recbuf; - uint32_t dmaster; - uint32_t reqid; - bool persistent; - bool failed; -}; - -static int recdb_traverse(struct tdb_context *tdb, TDB_DATA key, TDB_DATA data, - void *private_data) +/* This function decides which records from recdb are retained */ +static int recbuf_filter_add(struct ctdb_rec_buffer *recbuf, bool persistent, + uint32_t reqid, uint32_t dmaster, + TDB_DATA key, TDB_DATA data) { - struct recdb_traverse_state *state = - (struct recdb_traverse_state *)private_data; struct ctdb_ltdb_header *header; int ret; @@ -267,20 +259,43 @@ static int recdb_traverse(struct tdb_context *tdb, TDB_DATA key, TDB_DATA data, * On databases like Samba's registry, this can damage the higher-level * data structures built from the various tdb-level records. */ - if (!state->persistent && - data.dsize <= sizeof(struct ctdb_ltdb_header)) { + if (!persistent && data.dsize <= sizeof(struct ctdb_ltdb_header)) { return 0; } /* update the dmaster field to point to us */ header = (struct ctdb_ltdb_header *)data.dptr; - if (!state->persistent) { - header->dmaster = state->dmaster; + if (!persistent) { + header->dmaster = dmaster; header->flags |= CTDB_REC_FLAG_MIGRATED_WITH_DATA; } - ret = ctdb_rec_buffer_add(state->recbuf, state->recbuf, state->reqid, - NULL, key, data); + ret = ctdb_rec_buffer_add(recbuf, recbuf, reqid, NULL, key, data); + if (ret != 0) { + return ret; + } + + return 0; +} + +struct recdb_records_traverse_state { + struct ctdb_rec_buffer *recbuf; + uint32_t dmaster; + uint32_t reqid; + bool persistent; + bool failed; +}; + +static int recdb_records_traverse(struct tdb_context *tdb, + TDB_DATA key, TDB_DATA data, + void *private_data) +{ + struct recdb_records_traverse_state *state = + (struct recdb_records_traverse_state *)private_data; + int ret; + + ret = recbuf_filter_add(state->recbuf, state->persistent, + state->reqid, state->dmaster, key, data); if (ret != 0) { state->failed = true; return ret; @@ -293,7 +308,7 @@ static struct ctdb_rec_buffer *recdb_records(struct recdb_context *recdb, TALLOC_CTX *mem_ctx, uint32_t dmaster) { - struct recdb_traverse_state state; + struct recdb_records_traverse_state state; int ret; state.recbuf = ctdb_rec_buffer_init(mem_ctx, recdb_id(recdb)); @@ -305,7 +320,8 @@ static struct ctdb_rec_buffer *recdb_records(struct recdb_context *recdb, state.persistent = recdb_persistent(recdb); state.failed = false; - ret = tdb_traverse_read(recdb_tdb(recdb), recdb_traverse, &state); + ret = tdb_traverse_read(recdb_tdb(recdb), recdb_records_traverse, + &state); if (ret == -1 || state.failed) { LOG("Failed to marshall recovery records for %s\n", recdb_name(recdb)); -- 2.5.5 From 70db289e2c2ee375f3887f73fb9ebdab6a55018e Mon Sep 17 00:00:00 2001 From: Amitay Isaacs Date: Tue, 8 Mar 2016 18:32:19 +1100 Subject: [PATCH 20/29] ctdb-recovery-helper: Write recovery records to a recovery file Signed-off-by: Amitay Isaacs Reviewed-by: Martin Schwenke (cherry picked from commit e1fdfdd1c13b0a81e4d2e8b87aeb6f6bafef1a92) --- ctdb/server/ctdb_recovery_helper.c | 89 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 89 insertions(+) diff --git a/ctdb/server/ctdb_recovery_helper.c b/ctdb/server/ctdb_recovery_helper.c index 05900ef..78fb3f9 100644 --- a/ctdb/server/ctdb_recovery_helper.c +++ b/ctdb/server/ctdb_recovery_helper.c @@ -332,6 +332,95 @@ static struct ctdb_rec_buffer *recdb_records(struct recdb_context *recdb, return state.recbuf; } +struct recdb_file_traverse_state { + struct ctdb_rec_buffer *recbuf; + struct recdb_context *recdb; + TALLOC_CTX *mem_ctx; + uint32_t dmaster; + uint32_t reqid; + bool persistent; + bool failed; + int fd; + int max_size; + int num_buffers; +}; + +static int recdb_file_traverse(struct tdb_context *tdb, + TDB_DATA key, TDB_DATA data, + void *private_data) +{ + struct recdb_file_traverse_state *state = + (struct recdb_file_traverse_state *)private_data; + int ret; + + ret = recbuf_filter_add(state->recbuf, state->persistent, + state->reqid, state->dmaster, key, data); + if (ret != 0) { + state->failed = true; + return ret; + } + + if (ctdb_rec_buffer_len(state->recbuf) > state->max_size) { + ret = ctdb_rec_buffer_write(state->recbuf, state->fd); + if (ret != 0) { + LOG("Failed to collect recovery records for %s\n", + recdb_name(state->recdb)); + state->failed = true; + return ret; + } + + state->num_buffers += 1; + + TALLOC_FREE(state->recbuf); + state->recbuf = ctdb_rec_buffer_init(state->mem_ctx, + recdb_id(state->recdb)); + if (state->recbuf == NULL) { + state->failed = true; + return ENOMEM; + } + } + + return 0; +} + +static int recdb_file(struct recdb_context *recdb, TALLOC_CTX *mem_ctx, + uint32_t dmaster, int fd, int max_size) +{ + struct recdb_file_traverse_state state; + int ret; + + state.recbuf = ctdb_rec_buffer_init(mem_ctx, recdb_id(recdb)); + if (state.recbuf == NULL) { + return -1; + } + state.recdb = recdb; + state.mem_ctx = mem_ctx; + state.dmaster = dmaster; + state.reqid = 0; + state.persistent = recdb_persistent(recdb); + state.failed = false; + state.fd = fd; + state.max_size = max_size; + state.num_buffers = 0; + + ret = tdb_traverse_read(recdb_tdb(recdb), recdb_file_traverse, &state); + if (ret == -1 || state.failed) { + TALLOC_FREE(state.recbuf); + return -1; + } + + ret = ctdb_rec_buffer_write(state.recbuf, fd); + if (ret != 0) { + LOG("Failed to collect recovery records for %s\n", + recdb_name(recdb)); + TALLOC_FREE(state.recbuf); + return -1; + } + state.num_buffers += 1; + + return state.num_buffers; +} + /* * Collect databases using highest sequence number */ -- 2.5.5 From 0b40262231315376971455143b5342d84932bad7 Mon Sep 17 00:00:00 2001 From: Amitay Isaacs Date: Tue, 23 Feb 2016 18:03:10 +1100 Subject: [PATCH 21/29] ctdb-protocol: Add srvid for messages during recovery Signed-off-by: Amitay Isaacs Reviewed-by: Martin Schwenke (cherry picked from commit c5776f0529dd23979dd3ef36ffe7d7b8eb444d0d) --- ctdb/protocol/protocol.h | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ctdb/protocol/protocol.h b/ctdb/protocol/protocol.h index 16d1801..796b7e5 100644 --- a/ctdb/protocol/protocol.h +++ b/ctdb/protocol/protocol.h @@ -121,6 +121,9 @@ struct ctdb_call { /* SRVID prefix used by CTDB */ #define CTDB_SRVID_PREFIX 0xF000000000000000LL +/* SRVID prefix used during recovery for pulling and pushing databases */ +#define CTDB_SRVID_RECOVERY 0xF001000000000000LL + /* SRVID to inform of election data */ #define CTDB_SRVID_ELECTION 0xF100000000000000LL -- 2.5.5 From 99a4dc5633b163302e9502d35a7c9d9303d61c1f Mon Sep 17 00:00:00 2001 From: Amitay Isaacs Date: Tue, 23 Feb 2016 14:52:51 +1100 Subject: [PATCH 22/29] ctdb-protocol: Add new capability Signed-off-by: Amitay Isaacs Reviewed-by: Martin Schwenke (cherry picked from commit bb6541b386c7dfcc6dcbbd4cea39dc1ff80fe578) --- ctdb/protocol/protocol.h | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ctdb/protocol/protocol.h b/ctdb/protocol/protocol.h index 796b7e5..798c928 100644 --- a/ctdb/protocol/protocol.h +++ b/ctdb/protocol/protocol.h @@ -717,8 +717,10 @@ struct ctdb_public_ip_list { * Node features */ #define CTDB_CAP_PARALLEL_RECOVERY 0x00010000 +#define CTDB_CAP_FRAGMENTED_CONTROLS 0x00020000 -#define CTDB_CAP_FEATURES (CTDB_CAP_PARALLEL_RECOVERY) +#define CTDB_CAP_FEATURES (CTDB_CAP_PARALLEL_RECOVERY | \ + CTDB_CAP_FRAGMENTED_CONTROLS) #define CTDB_CAP_DEFAULT (CTDB_CAP_RECMASTER | \ CTDB_CAP_LMASTER | \ -- 2.5.5 From e4269bf89b75eea37691c82e67607ffe1a6c57bb Mon Sep 17 00:00:00 2001 From: Amitay Isaacs Date: Wed, 24 Feb 2016 18:10:49 +1100 Subject: [PATCH 23/29] ctdb-recovery-helper: Introduce pull database abstraction This abstraction depending on the capability of the remote node either uses older PULL_DB control or newer DB_PULL control. Signed-off-by: Amitay Isaacs Reviewed-by: Martin Schwenke (cherry picked from commit b96a4759b397d873d56ccdd0c0b26e770cc10b89) --- ctdb/server/ctdb_recovery_helper.c | 349 ++++++++++++++++++++++++++++++------- 1 file changed, 282 insertions(+), 67 deletions(-) diff --git a/ctdb/server/ctdb_recovery_helper.c b/ctdb/server/ctdb_recovery_helper.c index 78fb3f9..32fadc0 100644 --- a/ctdb/server/ctdb_recovery_helper.c +++ b/ctdb/server/ctdb_recovery_helper.c @@ -79,6 +79,14 @@ static bool generic_recv(struct tevent_req *req, int *perr) return true; } +static uint64_t rec_srvid = CTDB_SRVID_RECOVERY; + +static uint64_t srvid_next(void) +{ + rec_srvid += 1; + return rec_srvid; +} + /* * Recovery database functions */ @@ -422,6 +430,265 @@ static int recdb_file(struct recdb_context *recdb, TALLOC_CTX *mem_ctx, } /* + * Pull database from a single node + */ + +struct pull_database_state { + struct tevent_context *ev; + struct ctdb_client_context *client; + struct recdb_context *recdb; + uint32_t pnn; + uint64_t srvid; + int num_records; +}; + +static void pull_database_handler(uint64_t srvid, TDB_DATA data, + void *private_data); +static void pull_database_register_done(struct tevent_req *subreq); +static void pull_database_old_done(struct tevent_req *subreq); +static void pull_database_unregister_done(struct tevent_req *subreq); +static void pull_database_new_done(struct tevent_req *subreq); + +static struct tevent_req *pull_database_send( + TALLOC_CTX *mem_ctx, + struct tevent_context *ev, + struct ctdb_client_context *client, + uint32_t pnn, uint32_t caps, + struct recdb_context *recdb) +{ + struct tevent_req *req, *subreq; + struct pull_database_state *state; + struct ctdb_req_control request; + + req = tevent_req_create(mem_ctx, &state, struct pull_database_state); + if (req == NULL) { + return NULL; + } + + state->ev = ev; + state->client = client; + state->recdb = recdb; + state->pnn = pnn; + state->srvid = srvid_next(); + + if (caps & CTDB_CAP_FRAGMENTED_CONTROLS) { + subreq = ctdb_client_set_message_handler_send( + state, state->ev, state->client, + state->srvid, pull_database_handler, + req); + if (tevent_req_nomem(subreq, req)) { + return tevent_req_post(req, ev); + } + + tevent_req_set_callback(subreq, pull_database_register_done, + req); + + } else { + struct ctdb_pulldb pulldb; + + pulldb.db_id = recdb_id(recdb); + pulldb.lmaster = CTDB_LMASTER_ANY; + + ctdb_req_control_pull_db(&request, &pulldb); + subreq = ctdb_client_control_send(state, state->ev, + state->client, + pnn, TIMEOUT(), + &request); + if (tevent_req_nomem(subreq, req)) { + return tevent_req_post(req, ev); + } + tevent_req_set_callback(subreq, pull_database_old_done, req); + } + + return req; +} + +static void pull_database_handler(uint64_t srvid, TDB_DATA data, + void *private_data) +{ + struct tevent_req *req = talloc_get_type_abort( + private_data, struct tevent_req); + struct pull_database_state *state = tevent_req_data( + req, struct pull_database_state); + struct ctdb_rec_buffer *recbuf; + int ret; + bool status; + + if (srvid != state->srvid) { + return; + } + + ret = ctdb_rec_buffer_pull(data.dptr, data.dsize, state, &recbuf); + if (ret != 0) { + LOG("Invalid data received for DB_PULL messages\n"); + return; + } + + if (recbuf->db_id != recdb_id(state->recdb)) { + talloc_free(recbuf); + LOG("Invalid dbid:%08x for DB_PULL messages for %s\n", + recbuf->db_id, recdb_name(state->recdb)); + return; + } + + status = recdb_add(state->recdb, ctdb_client_pnn(state->client), + recbuf); + if (! status) { + talloc_free(recbuf); + LOG("Failed to add records to recdb for %s\n", + recdb_name(state->recdb)); + return; + } + + state->num_records += recbuf->count; + talloc_free(recbuf); +} + +static void pull_database_register_done(struct tevent_req *subreq) +{ + struct tevent_req *req = tevent_req_callback_data( + subreq, struct tevent_req); + struct pull_database_state *state = tevent_req_data( + req, struct pull_database_state); + struct ctdb_req_control request; + struct ctdb_pulldb_ext pulldb_ext; + int ret; + bool status; + + status = ctdb_client_set_message_handler_recv(subreq, &ret); + TALLOC_FREE(subreq); + if (! status) { + LOG("failed to set message handler for DB_PULL for %s\n", + recdb_name(state->recdb)); + tevent_req_error(req, ret); + return; + } + + pulldb_ext.db_id = recdb_id(state->recdb); + pulldb_ext.lmaster = CTDB_LMASTER_ANY; + pulldb_ext.srvid = state->srvid; + + ctdb_req_control_db_pull(&request, &pulldb_ext); + subreq = ctdb_client_control_send(state, state->ev, state->client, + state->pnn, TIMEOUT(), &request); + if (tevent_req_nomem(subreq, req)) { + return; + } + tevent_req_set_callback(subreq, pull_database_new_done, req); +} + +static void pull_database_old_done(struct tevent_req *subreq) +{ + struct tevent_req *req = tevent_req_callback_data( + subreq, struct tevent_req); + struct pull_database_state *state = tevent_req_data( + req, struct pull_database_state); + struct ctdb_reply_control *reply; + struct ctdb_rec_buffer *recbuf; + int ret; + bool status; + + status = ctdb_client_control_recv(subreq, &ret, state, &reply); + TALLOC_FREE(subreq); + if (! status) { + LOG("control PULL_DB failed for %s on node %u, ret=%d\n", + recdb_name(state->recdb), state->pnn, ret); + tevent_req_error(req, ret); + return; + } + + ret = ctdb_reply_control_pull_db(reply, state, &recbuf); + talloc_free(reply); + if (ret != 0) { + tevent_req_error(req, ret); + return; + } + + status = recdb_add(state->recdb, ctdb_client_pnn(state->client), + recbuf); + if (! status) { + talloc_free(recbuf); + tevent_req_error(req, EIO); + return; + } + + state->num_records = recbuf->count; + talloc_free(recbuf); + + LOG("Pulled %d records for db %s from node %d\n", + state->num_records, recdb_name(state->recdb), state->pnn); + + tevent_req_done(req); +} + +static void pull_database_new_done(struct tevent_req *subreq) +{ + struct tevent_req *req = tevent_req_callback_data( + subreq, struct tevent_req); + struct pull_database_state *state = tevent_req_data( + req, struct pull_database_state); + struct ctdb_reply_control *reply; + uint32_t num_records; + int ret; + bool status; + + status = ctdb_client_control_recv(subreq, &ret, state, &reply); + TALLOC_FREE(subreq); + if (! status) { + LOG("control DB_PULL failed for %s on node %u, ret=%d\n", + recdb_name(state->recdb), state->pnn, ret); + tevent_req_error(req, ret); + return; + } + + ret = ctdb_reply_control_db_pull(reply, &num_records); + talloc_free(reply); + if (num_records != state->num_records) { + LOG("mismatch (%u != %u) in DB_PULL records for %s\n", + num_records, state->num_records, recdb_name(state->recdb)); + tevent_req_error(req, EIO); + return; + } + + LOG("Pulled %d records for db %s from node %d\n", + state->num_records, recdb_name(state->recdb), state->pnn); + + subreq = ctdb_client_remove_message_handler_send( + state, state->ev, state->client, + state->srvid, req); + if (tevent_req_nomem(subreq, req)) { + return; + } + tevent_req_set_callback(subreq, pull_database_unregister_done, req); +} + +static void pull_database_unregister_done(struct tevent_req *subreq) +{ + struct tevent_req *req = tevent_req_callback_data( + subreq, struct tevent_req); + struct pull_database_state *state = tevent_req_data( + req, struct pull_database_state); + int ret; + bool status; + + status = ctdb_client_remove_message_handler_recv(subreq, &ret); + TALLOC_FREE(subreq); + if (! status) { + LOG("failed to remove message handler for DB_PULL for %s\n", + recdb_name(state->recdb)); + tevent_req_error(req, ret); + return; + } + + tevent_req_done(req); +} + +static bool pull_database_recv(struct tevent_req *req, int *perr) +{ + return generic_recv(req, perr); +} + +/* * Collect databases using highest sequence number */ @@ -484,8 +751,6 @@ static void collect_highseqnum_db_seqnum_done(struct tevent_req *subreq) struct collect_highseqnum_db_state *state = tevent_req_data( req, struct collect_highseqnum_db_state); struct ctdb_reply_control **reply; - struct ctdb_req_control request; - struct ctdb_pulldb pulldb; int *err_list; bool status; int ret, i; @@ -532,12 +797,10 @@ static void collect_highseqnum_db_seqnum_done(struct tevent_req *subreq) LOG("Pull persistent db %s from node %d with seqnum 0x%"PRIx64"\n", recdb_name(state->recdb), state->max_pnn, max_seqnum); - pulldb.db_id = state->db_id; - pulldb.lmaster = CTDB_LMASTER_ANY; - - ctdb_req_control_pull_db(&request, &pulldb); - subreq = ctdb_client_control_send(state, state->ev, state->client, - state->max_pnn, TIMEOUT(), &request); + subreq = pull_database_send(state, state->ev, state->client, + state->max_pnn, + state->caps[state->max_pnn], + state->recdb); if (tevent_req_nomem(subreq, req)) { return; } @@ -549,37 +812,16 @@ static void collect_highseqnum_db_pulldb_done(struct tevent_req *subreq) { struct tevent_req *req = tevent_req_callback_data( subreq, struct tevent_req); - struct collect_highseqnum_db_state *state = tevent_req_data( - req, struct collect_highseqnum_db_state); - struct ctdb_reply_control *reply; - struct ctdb_rec_buffer *recbuf; int ret; bool status; - status = ctdb_client_control_recv(subreq, &ret, state, &reply); + status = pull_database_recv(subreq, &ret); TALLOC_FREE(subreq); if (! status) { - LOG("control PULL_DB failed for %s on node %u, ret=%d\n", - recdb_name(state->recdb), state->max_pnn, ret); tevent_req_error(req, ret); return; } - ret = ctdb_reply_control_pull_db(reply, state, &recbuf); - if (ret != 0) { - tevent_req_error(req, EPROTO); - return; - } - - talloc_free(reply); - - ret = recdb_add(state->recdb, ctdb_client_pnn(state->client), recbuf); - talloc_free(recbuf); - if (! ret) { - tevent_req_error(req, EIO); - return; - } - tevent_req_done(req); } @@ -615,7 +857,7 @@ static struct tevent_req *collect_all_db_send( { struct tevent_req *req, *subreq; struct collect_all_db_state *state; - struct ctdb_req_control request; + uint32_t pnn; req = tevent_req_create(mem_ctx, &state, struct collect_all_db_state); @@ -627,18 +869,14 @@ static struct tevent_req *collect_all_db_send( state->client = client; state->pnn_list = pnn_list; state->count = count; + state->caps = caps; state->db_id = db_id; state->recdb = recdb; - - state->pulldb.db_id = db_id; - state->pulldb.lmaster = CTDB_LMASTER_ANY; - state->index = 0; - ctdb_req_control_pull_db(&request, &state->pulldb); - subreq = ctdb_client_control_send(state, ev, client, - state->pnn_list[state->index], - TIMEOUT(), &request); + pnn = state->pnn_list[state->index]; + + subreq = pull_database_send(state, ev, client, pnn, caps[pnn], recdb); if (tevent_req_nomem(subreq, req)) { return tevent_req_post(req, ev); } @@ -653,49 +891,26 @@ static void collect_all_db_pulldb_done(struct tevent_req *subreq) subreq, struct tevent_req); struct collect_all_db_state *state = tevent_req_data( req, struct collect_all_db_state); - struct ctdb_reply_control *reply; - struct ctdb_req_control request; - struct ctdb_rec_buffer *recbuf; + uint32_t pnn; int ret; bool status; - status = ctdb_client_control_recv(subreq, &ret, state, &reply); + status = pull_database_recv(subreq, &ret); TALLOC_FREE(subreq); if (! status) { - LOG("control PULL_DB failed for %s from node %u, ret=%d\n", - recdb_name(state->recdb), state->pnn_list[state->index], - ret); tevent_req_error(req, ret); return; } - ret = ctdb_reply_control_pull_db(reply, state, &recbuf); - if (ret != 0) { - LOG("control PULL_DB failed for %s, ret=%d\n", - recdb_name(state->recdb), ret); - tevent_req_error(req, EPROTO); - return; - } - - talloc_free(reply); - - status = recdb_add(state->recdb, ctdb_client_pnn(state->client), recbuf); - talloc_free(recbuf); - if (! status) { - tevent_req_error(req, EIO); - return; - } - state->index += 1; if (state->index == state->count) { tevent_req_done(req); return; } - ctdb_req_control_pull_db(&request, &state->pulldb); - subreq = ctdb_client_control_send(state, state->ev, state->client, - state->pnn_list[state->index], - TIMEOUT(), &request); + pnn = state->pnn_list[state->index]; + subreq = pull_database_send(state, state->ev, state->client, + pnn, state->caps[pnn], state->recdb); if (tevent_req_nomem(subreq, req)) { return; } -- 2.5.5 From 19ed1f6743af77d69f1740d1b6a474c93c7bfb4c Mon Sep 17 00:00:00 2001 From: Amitay Isaacs Date: Thu, 25 Feb 2016 18:07:11 +1100 Subject: [PATCH 24/29] ctdb-recovery-helper: Introduce push database abstraction This abstraction uses capabilities of the remote nodes to either send older PUSH_DB controls or newer DB_PUSH_START and DB_PUSH_CONFIRM controls. Signed-off-by: Amitay Isaacs Reviewed-by: Martin Schwenke (cherry picked from commit ffea827bae2a8054ad488ae82eedb021cdfb71c4) --- ctdb/server/ctdb_recovery_helper.c | 556 ++++++++++++++++++++++++++++++++++--- 1 file changed, 524 insertions(+), 32 deletions(-) diff --git a/ctdb/server/ctdb_recovery_helper.c b/ctdb/server/ctdb_recovery_helper.c index 32fadc0..9991d31 100644 --- a/ctdb/server/ctdb_recovery_helper.c +++ b/ctdb/server/ctdb_recovery_helper.c @@ -426,6 +426,9 @@ static int recdb_file(struct recdb_context *recdb, TALLOC_CTX *mem_ctx, } state.num_buffers += 1; + LOG("Wrote %d buffers of recovery records for %s\n", + state.num_buffers, recdb_name(recdb)); + return state.num_buffers; } @@ -689,6 +692,521 @@ static bool pull_database_recv(struct tevent_req *req, int *perr) } /* + * Push database to specified nodes (old style) + */ + +struct push_database_old_state { + struct tevent_context *ev; + struct ctdb_client_context *client; + struct recdb_context *recdb; + uint32_t *pnn_list; + int count; + struct ctdb_rec_buffer *recbuf; + int index; +}; + +static void push_database_old_push_done(struct tevent_req *subreq); + +static struct tevent_req *push_database_old_send( + TALLOC_CTX *mem_ctx, + struct tevent_context *ev, + struct ctdb_client_context *client, + uint32_t *pnn_list, int count, + struct recdb_context *recdb) +{ + struct tevent_req *req, *subreq; + struct push_database_old_state *state; + struct ctdb_req_control request; + uint32_t pnn; + + req = tevent_req_create(mem_ctx, &state, + struct push_database_old_state); + if (req == NULL) { + return NULL; + } + + state->ev = ev; + state->client = client; + state->recdb = recdb; + state->pnn_list = pnn_list; + state->count = count; + state->index = 0; + + state->recbuf = recdb_records(recdb, state, + ctdb_client_pnn(client)); + if (tevent_req_nomem(state->recbuf, req)) { + return tevent_req_post(req, ev); + } + + pnn = state->pnn_list[state->index]; + + ctdb_req_control_push_db(&request, state->recbuf); + subreq = ctdb_client_control_send(state, ev, client, pnn, + TIMEOUT(), &request); + if (tevent_req_nomem(subreq, req)) { + return tevent_req_post(req, ev); + } + tevent_req_set_callback(subreq, push_database_old_push_done, req); + + return req; +} + +static void push_database_old_push_done(struct tevent_req *subreq) +{ + struct tevent_req *req = tevent_req_callback_data( + subreq, struct tevent_req); + struct push_database_old_state *state = tevent_req_data( + req, struct push_database_old_state); + struct ctdb_req_control request; + uint32_t pnn; + int ret; + bool status; + + status = ctdb_client_control_recv(subreq, &ret, NULL, NULL); + TALLOC_FREE(subreq); + if (! status) { + LOG("control PUSH_DB failed for db %s on node %u, ret=%d\n", + recdb_name(state->recdb), state->pnn_list[state->index], + ret); + tevent_req_error(req, ret); + return; + } + + state->index += 1; + if (state->index == state->count) { + TALLOC_FREE(state->recbuf); + tevent_req_done(req); + return; + } + + pnn = state->pnn_list[state->index]; + + ctdb_req_control_push_db(&request, state->recbuf); + subreq = ctdb_client_control_send(state, state->ev, state->client, + pnn, TIMEOUT(), &request); + if (tevent_req_nomem(subreq, req)) { + return; + } + tevent_req_set_callback(subreq, push_database_old_push_done, req); +} + +static bool push_database_old_recv(struct tevent_req *req, int *perr) +{ + return generic_recv(req, perr); +} + +/* + * Push database to specified nodes (new style) + */ + +struct push_database_new_state { + struct tevent_context *ev; + struct ctdb_client_context *client; + struct recdb_context *recdb; + uint32_t *pnn_list; + int count; + uint64_t srvid; + uint32_t dmaster; + int fd; + int num_buffers; + int num_buffers_sent; + int num_records; +}; + +static void push_database_new_started(struct tevent_req *subreq); +static void push_database_new_send_msg(struct tevent_req *req); +static void push_database_new_send_done(struct tevent_req *subreq); +static void push_database_new_confirmed(struct tevent_req *subreq); + +static struct tevent_req *push_database_new_send( + TALLOC_CTX *mem_ctx, + struct tevent_context *ev, + struct ctdb_client_context *client, + uint32_t *pnn_list, int count, + struct recdb_context *recdb, + int max_size) +{ + struct tevent_req *req, *subreq; + struct push_database_new_state *state; + struct ctdb_req_control request; + struct ctdb_pulldb_ext pulldb_ext; + char *filename; + off_t offset; + + req = tevent_req_create(mem_ctx, &state, + struct push_database_new_state); + if (req == NULL) { + return NULL; + } + + state->ev = ev; + state->client = client; + state->recdb = recdb; + state->pnn_list = pnn_list; + state->count = count; + + state->srvid = srvid_next(); + state->dmaster = ctdb_client_pnn(client); + state->num_buffers_sent = 0; + state->num_records = 0; + + filename = talloc_asprintf(state, "%s.dat", recdb_path(recdb)); + if (tevent_req_nomem(filename, req)) { + return tevent_req_post(req, ev); + } + + state->fd = open(filename, O_RDWR|O_CREAT, 0644); + if (state->fd == -1) { + tevent_req_error(req, errno); + return tevent_req_post(req, ev); + } + unlink(filename); + talloc_free(filename); + + state->num_buffers = recdb_file(recdb, state, state->dmaster, + state->fd, max_size); + if (state->num_buffers == -1) { + tevent_req_error(req, ENOMEM); + return tevent_req_post(req, ev); + } + + offset = lseek(state->fd, 0, SEEK_SET); + if (offset != 0) { + tevent_req_error(req, EIO); + return tevent_req_post(req, ev); + } + + pulldb_ext.db_id = recdb_id(recdb); + pulldb_ext.srvid = state->srvid; + + ctdb_req_control_db_push_start(&request, &pulldb_ext); + subreq = ctdb_client_control_multi_send(state, ev, client, + pnn_list, count, + TIMEOUT(), &request); + if (tevent_req_nomem(subreq, req)) { + return tevent_req_post(req, ev); + } + tevent_req_set_callback(subreq, push_database_new_started, req); + + return req; +} + +static void push_database_new_started(struct tevent_req *subreq) +{ + struct tevent_req *req = tevent_req_callback_data( + subreq, struct tevent_req); + struct push_database_new_state *state = tevent_req_data( + req, struct push_database_new_state); + int *err_list; + int ret; + bool status; + + status = ctdb_client_control_multi_recv(subreq, &ret, state, + &err_list, NULL); + TALLOC_FREE(subreq); + if (! status) { + int ret2; + uint32_t pnn; + + ret2 = ctdb_client_control_multi_error(state->pnn_list, + state->count, + err_list, &pnn); + if (ret2 != 0) { + LOG("control DB_PUSH_START failed for db %s " + "on node %u, ret=%d\n", + recdb_name(state->recdb), pnn, ret2); + } else { + LOG("control DB_PUSH_START failed for db %s, ret=%d\n", + recdb_name(state->recdb), ret); + } + talloc_free(err_list); + + tevent_req_error(req, ret); + return; + } + + push_database_new_send_msg(req); +} + +static void push_database_new_send_msg(struct tevent_req *req) +{ + struct push_database_new_state *state = tevent_req_data( + req, struct push_database_new_state); + struct tevent_req *subreq; + struct ctdb_rec_buffer *recbuf; + struct ctdb_req_message message; + TDB_DATA data; + int ret; + + if (state->num_buffers_sent == state->num_buffers) { + struct ctdb_req_control request; + + ctdb_req_control_db_push_confirm(&request, + recdb_id(state->recdb)); + subreq = ctdb_client_control_multi_send(state, state->ev, + state->client, + state->pnn_list, + state->count, + TIMEOUT(), &request); + if (tevent_req_nomem(subreq, req)) { + return; + } + tevent_req_set_callback(subreq, push_database_new_confirmed, + req); + return; + } + + ret = ctdb_rec_buffer_read(state->fd, state, &recbuf); + if (ret != 0) { + tevent_req_error(req, ret); + return; + } + + data.dsize = ctdb_rec_buffer_len(recbuf); + data.dptr = talloc_size(state, data.dsize); + if (tevent_req_nomem(data.dptr, req)) { + return; + } + + ctdb_rec_buffer_push(recbuf, data.dptr); + + message.srvid = state->srvid; + message.data.data = data; + + LOG("Pushing buffer %d with %d records for %s\n", + state->num_buffers_sent, recbuf->count, recdb_name(state->recdb)); + + subreq = ctdb_client_message_multi_send(state, state->ev, + state->client, + state->pnn_list, state->count, + &message); + if (tevent_req_nomem(subreq, req)) { + return; + } + tevent_req_set_callback(subreq, push_database_new_send_done, req); + + state->num_records += recbuf->count; + + talloc_free(data.dptr); + talloc_free(recbuf); +} + +static void push_database_new_send_done(struct tevent_req *subreq) +{ + struct tevent_req *req = tevent_req_callback_data( + subreq, struct tevent_req); + struct push_database_new_state *state = tevent_req_data( + req, struct push_database_new_state); + bool status; + int ret; + + status = ctdb_client_message_multi_recv(subreq, &ret, NULL, NULL); + TALLOC_FREE(subreq); + if (! status) { + LOG("Sending recovery records failed for %s\n", + recdb_name(state->recdb)); + tevent_req_error(req, ret); + return; + } + + state->num_buffers_sent += 1; + + push_database_new_send_msg(req); +} + +static void push_database_new_confirmed(struct tevent_req *subreq) +{ + struct tevent_req *req = tevent_req_callback_data( + subreq, struct tevent_req); + struct push_database_new_state *state = tevent_req_data( + req, struct push_database_new_state); + struct ctdb_reply_control **reply; + int *err_list; + bool status; + int ret, i; + uint32_t num_records; + + status = ctdb_client_control_multi_recv(subreq, &ret, state, + &err_list, &reply); + TALLOC_FREE(subreq); + if (! status) { + int ret2; + uint32_t pnn; + + ret2 = ctdb_client_control_multi_error(state->pnn_list, + state->count, err_list, + &pnn); + if (ret2 != 0) { + LOG("control DB_PUSH_CONFIRM failed for %s on node %u," + " ret=%d\n", recdb_name(state->recdb), pnn, ret2); + } else { + LOG("control DB_PUSH_CONFIRM failed for %s, ret=%d\n", + recdb_name(state->recdb), ret); + } + tevent_req_error(req, ret); + return; + } + + for (i=0; icount; i++) { + ret = ctdb_reply_control_db_push_confirm(reply[i], + &num_records); + if (ret != 0) { + tevent_req_error(req, EPROTO); + return; + } + + if (num_records != state->num_records) { + LOG("Node %u received %d of %d records for %s\n", + state->pnn_list[i], num_records, + state->num_records, recdb_name(state->recdb)); + tevent_req_error(req, EPROTO); + return; + } + } + + talloc_free(reply); + + LOG("Pushed %d records for db %s\n", + state->num_records, recdb_name(state->recdb)); + + tevent_req_done(req); +} + +static bool push_database_new_recv(struct tevent_req *req, int *perr) +{ + return generic_recv(req, perr); +} + +/* + * wrapper for push_database_old and push_database_new + */ + +struct push_database_state { + bool old_done, new_done; +}; + +static void push_database_old_done(struct tevent_req *subreq); +static void push_database_new_done(struct tevent_req *subreq); + +static struct tevent_req *push_database_send( + TALLOC_CTX *mem_ctx, + struct tevent_context *ev, + struct ctdb_client_context *client, + uint32_t *pnn_list, int count, uint32_t *caps, + struct ctdb_tunable_list *tun_list, + struct recdb_context *recdb) +{ + struct tevent_req *req, *subreq; + struct push_database_state *state; + uint32_t *old_list, *new_list; + int old_count, new_count; + int i; + + req = tevent_req_create(mem_ctx, &state, struct push_database_state); + if (req == NULL) { + return NULL; + } + + state->old_done = false; + state->new_done = false; + + old_count = 0; + new_count = 0; + old_list = talloc_array(state, uint32_t, count); + new_list = talloc_array(state, uint32_t, count); + if (tevent_req_nomem(old_list, req) || + tevent_req_nomem(new_list,req)) { + return tevent_req_post(req, ev); + } + + for (i=0; i 0) { + subreq = push_database_old_send(state, ev, client, + old_list, old_count, recdb); + if (tevent_req_nomem(subreq, req)) { + return tevent_req_post(req, ev); + } + tevent_req_set_callback(subreq, push_database_old_done, req); + } else { + state->old_done = true; + } + + if (new_count > 0) { + subreq = push_database_new_send(state, ev, client, + new_list, new_count, recdb, + tun_list->rec_buffer_size_limit); + if (tevent_req_nomem(subreq, req)) { + return tevent_req_post(req, ev); + } + tevent_req_set_callback(subreq, push_database_new_done, req); + } else { + state->new_done = true; + } + + return req; +} + +static void push_database_old_done(struct tevent_req *subreq) +{ + struct tevent_req *req = tevent_req_callback_data( + subreq, struct tevent_req); + struct push_database_state *state = tevent_req_data( + req, struct push_database_state); + bool status; + int ret; + + status = push_database_old_recv(subreq, &ret); + if (! status) { + tevent_req_error(req, ret); + return; + } + + state->old_done = true; + + if (state->old_done && state->new_done) { + tevent_req_done(req); + } +} + +static void push_database_new_done(struct tevent_req *subreq) +{ + struct tevent_req *req = tevent_req_callback_data( + subreq, struct tevent_req); + struct push_database_state *state = tevent_req_data( + req, struct push_database_state); + bool status; + int ret; + + status = push_database_new_recv(subreq, &ret); + if (! status) { + tevent_req_error(req, ret); + return; + } + + state->new_done = true; + + if (state->old_done && state->new_done) { + tevent_req_done(req); + } +} + +static bool push_database_recv(struct tevent_req *req, int *perr) +{ + return generic_recv(req, perr); +} + +/* * Collect databases using highest sequence number */ @@ -951,8 +1469,6 @@ struct recover_db_state { const char *db_name, *db_path; struct recdb_context *recdb; - struct ctdb_rec_buffer *recbuf; - }; static void recover_db_name_done(struct tevent_req *subreq); @@ -1225,7 +1741,6 @@ static void recover_db_wipedb_done(struct tevent_req *subreq) subreq, struct tevent_req); struct recover_db_state *state = tevent_req_data( req, struct recover_db_state); - struct ctdb_req_control request; int *err_list; int ret; bool status; @@ -1251,18 +1766,10 @@ static void recover_db_wipedb_done(struct tevent_req *subreq) return; } - state->recbuf = recdb_records(state->recdb, state, state->destnode); - if (tevent_req_nomem(state->recbuf, req)) { - return; - } - - TALLOC_FREE(state->recdb); - - ctdb_req_control_push_db(&request, state->recbuf); - subreq = ctdb_client_control_multi_send(state, state->ev, - state->client, - state->pnn_list, state->count, - TIMEOUT(), &request); + subreq = push_database_send(state, state->ev, state->client, + state->pnn_list, state->count, + state->caps, state->tun_list, + state->recdb); if (tevent_req_nomem(subreq, req)) { return; } @@ -1276,32 +1783,17 @@ static void recover_db_pushdb_done(struct tevent_req *subreq) struct recover_db_state *state = tevent_req_data( req, struct recover_db_state); struct ctdb_req_control request; - int *err_list; int ret; bool status; - status = ctdb_client_control_multi_recv(subreq, &ret, NULL, &err_list, - NULL); + status = push_database_recv(subreq, &ret); TALLOC_FREE(subreq); if (! status) { - int ret2; - uint32_t pnn; - - ret2 = ctdb_client_control_multi_error(state->pnn_list, - state->count, - err_list, &pnn); - if (ret2 != 0) { - LOG("control PUSHDB failed for db %s on node %u," - " ret=%d\n", state->db_name, pnn, ret2); - } else { - LOG("control PUSHDB failed for db %s, ret=%d\n", - state->db_name, ret); - } tevent_req_error(req, ret); return; } - TALLOC_FREE(state->recbuf); + TALLOC_FREE(state->recdb); ctdb_req_control_db_transaction_commit(&request, &state->transdb); subreq = ctdb_client_control_multi_send(state, state->ev, -- 2.5.5 From 0cc13ecde2d16f925d2a6b3a1826d6828c7f0f93 Mon Sep 17 00:00:00 2001 From: Amitay Isaacs Date: Sat, 5 Mar 2016 15:04:48 +1100 Subject: [PATCH 25/29] ctdb-tests: Add a test for recovery of large databases Signed-off-by: Amitay Isaacs Reviewed-by: Martin Schwenke (cherry picked from commit a7b8ee87fe1b58ffb9da02ae8e3959a385e628d9) --- ctdb/tests/simple/78_ctdb_large_db_recovery.sh | 107 +++++++++++++++++++++++++ 1 file changed, 107 insertions(+) create mode 100755 ctdb/tests/simple/78_ctdb_large_db_recovery.sh diff --git a/ctdb/tests/simple/78_ctdb_large_db_recovery.sh b/ctdb/tests/simple/78_ctdb_large_db_recovery.sh new file mode 100755 index 0000000..51fbd25 --- /dev/null +++ b/ctdb/tests/simple/78_ctdb_large_db_recovery.sh @@ -0,0 +1,107 @@ +#!/bin/bash + +test_info() +{ + cat < Date: Thu, 17 Mar 2016 14:13:02 +1100 Subject: [PATCH 26/29] ctdb-recovery-helper: Improve log message Signed-off-by: Amitay Isaacs Reviewed-by: Martin Schwenke (cherry picked from commit e5a714a3c29fd8fdebe0203e68c7ce19f54af2d3) --- ctdb/server/ctdb_recovery_helper.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ctdb/server/ctdb_recovery_helper.c b/ctdb/server/ctdb_recovery_helper.c index 9991d31..c741c2c 100644 --- a/ctdb/server/ctdb_recovery_helper.c +++ b/ctdb/server/ctdb_recovery_helper.c @@ -2541,7 +2541,7 @@ static void recovery_db_recovery_done(struct tevent_req *subreq) status = db_recovery_recv(subreq, &count); TALLOC_FREE(subreq); - LOG("%d databases recovered\n", count); + LOG("%d of %d databases recovered\n", count, state->dbmap->num); if (! status) { tevent_req_error(req, EIO); -- 2.5.5 From df0b2a82f4badb77c1b93ad6df6443354d3b6284 Mon Sep 17 00:00:00 2001 From: Amitay Isaacs Date: Thu, 17 Mar 2016 14:22:17 +1100 Subject: [PATCH 27/29] ctdb-recovery-helper: Introduce new #define variable ... instead of hardcoding number of retries. Signed-off-by: Amitay Isaacs Reviewed-by: Martin Schwenke (cherry picked from commit ad7a407a13b87ec13d94a808111d2583bfd1d217) --- ctdb/server/ctdb_recovery_helper.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ctdb/server/ctdb_recovery_helper.c b/ctdb/server/ctdb_recovery_helper.c index c741c2c..5911644 100644 --- a/ctdb/server/ctdb_recovery_helper.c +++ b/ctdb/server/ctdb_recovery_helper.c @@ -36,6 +36,8 @@ static int recover_timeout = 120; +#define NUM_RETRIES 5 + #define TIMEOUT() timeval_current_ofs(recover_timeout, 0) static void LOG(const char *fmt, ...) @@ -1999,7 +2001,7 @@ static void db_recovery_one_done(struct tevent_req *subreq) } substate->num_fails += 1; - if (substate->num_fails < 5) { + if (substate->num_fails < NUM_RETRIES) { subreq = recover_db_send(state, state->ev, substate->client, substate->tun_list, substate->pnn_list, substate->count, -- 2.5.5 From aee8459b09215abc26f10aeb4187fee92374bd2f Mon Sep 17 00:00:00 2001 From: Amitay Isaacs Date: Thu, 2 Jun 2016 18:27:29 +1000 Subject: [PATCH 28/29] ctdb-recovery: Update timeout and number of retries during recovery The timeout RecoverTimeout (default 120) is used for control messages sent during the recovery. If any of the nodes does not respond to any of the recovery control messages for RecoverTimeout seconds, then it will cause a failure of recovery of a database. Recovery helper will retry the recovery for a database 5 times. In the worst case, if a database could not be recovered within 5 attempts, a total of 600 seconds would have passed. During this time period other timeouts will be triggered causing unnecessary failures as follows: 1. During the recovery, even though recoverd is processing events, it does not send a ping message to ctdb daemon. If a ping message is not received for RecdPingTimeout (default 60) seconds, then ctdb will count it as unresponsive recovery daemon. If the recovery daemon fails for RecdFailCount (default 10) times, then ctdb daemon will restart recovery daemon. So after 600 seconds, ctdb daemon will restart recovery daemon. 2. If ctdb daemon stays in recovery for RecoveryDropAllIPs (default 120), then it will drop all the public addresses. This will cause all SMB client to be disconnected unnecessarily. The released public addresses will not be taken over till the recovery is complete. To avoid dropping of IPs and restarting recovery daemon during a delayed recovery, adjust RecoverTimeout to 30 seconds and limit number of retries for recovering a database to 3. If we don't hear from a node for more than 25 seconds, then the node is considered disconnected. So 30 seconds is sufficient timeout for controls during recovery. Signed-off-by: Amitay Isaacs Reviewed-by: Martin Schwenke Autobuild-User(master): Martin Schwenke Autobuild-Date(master): Mon Jun 6 08:49:15 CEST 2016 on sn-devel-144 (cherry picked from commit 93dcca2a5f7af9698c9ba1024dbce1d1a66d4efb) --- ctdb/server/ctdb_recovery_helper.c | 4 ++-- ctdb/server/ctdb_tunables.c | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ctdb/server/ctdb_recovery_helper.c b/ctdb/server/ctdb_recovery_helper.c index 5911644..d1ec1e7 100644 --- a/ctdb/server/ctdb_recovery_helper.c +++ b/ctdb/server/ctdb_recovery_helper.c @@ -34,9 +34,9 @@ #include "protocol/protocol_api.h" #include "client/client.h" -static int recover_timeout = 120; +static int recover_timeout = 30; -#define NUM_RETRIES 5 +#define NUM_RETRIES 3 #define TIMEOUT() timeval_current_ofs(recover_timeout, 0) diff --git a/ctdb/server/ctdb_tunables.c b/ctdb/server/ctdb_tunables.c index 50c5c77..7cb1a24 100644 --- a/ctdb/server/ctdb_tunables.c +++ b/ctdb/server/ctdb_tunables.c @@ -41,7 +41,7 @@ static const struct { { "TraverseTimeout", 20, offsetof(struct ctdb_tunable_list, traverse_timeout), false }, { "KeepaliveInterval", 5, offsetof(struct ctdb_tunable_list, keepalive_interval), false }, { "KeepaliveLimit", 5, offsetof(struct ctdb_tunable_list, keepalive_limit), false }, - { "RecoverTimeout", 120, offsetof(struct ctdb_tunable_list, recover_timeout), false }, + { "RecoverTimeout", 30, offsetof(struct ctdb_tunable_list, recover_timeout), false }, { "RecoverInterval", 1, offsetof(struct ctdb_tunable_list, recover_interval), false }, { "ElectionTimeout", 3, offsetof(struct ctdb_tunable_list, election_timeout), false }, { "TakeoverTimeout", 9, offsetof(struct ctdb_tunable_list, takeover_timeout), false }, -- 2.5.5 From ff6780d50a5f0b3028db0ebc80a9751cdc41e87d Mon Sep 17 00:00:00 2001 From: Amitay Isaacs Date: Wed, 8 Jun 2016 15:04:52 +1000 Subject: [PATCH 29/29] ctdb-daemon: Reset push_started flag once DB_PUSH_CONFIRM is done Once DB_PUSH_START is processed as part of recovery, push_started flag tracks if there are multiple attempts to send DB_PUSH_START. In DB_PUSH_CONFIRM, once the record count is confirmed, all information related to DB_PUSH should be reset. However, The push_started flag was not reset when the push_state was reset. Signed-off-by: Amitay Isaacs Reviewed-by: Martin Schwenke Autobuild-User(master): Amitay Isaacs Autobuild-Date(master): Wed Jun 8 14:31:52 CEST 2016 on sn-devel-144 (cherry picked from commit c620bf5debd57a4a5d7f893a2b6383098ff7a919) --- ctdb/server/ctdb_recover.c | 1 + 1 file changed, 1 insertion(+) diff --git a/ctdb/server/ctdb_recover.c b/ctdb/server/ctdb_recover.c index 49f568a..47461f4 100644 --- a/ctdb/server/ctdb_recover.c +++ b/ctdb/server/ctdb_recover.c @@ -742,6 +742,7 @@ int32_t ctdb_control_db_push_confirm(struct ctdb_context *ctdb, outdata->dsize = sizeof(uint32_t); talloc_free(state); + ctdb_db->push_started = false; ctdb_db->push_state = NULL; return 0; -- 2.5.5