BUG#19795152: Wait for LCP start at node restart, add node recovery status table as well

Author: Mikael Ronstrom

mysql-test/suite/ndb/r/ndbinfo.result

@@ -65,9 +65,10 @@ table_id	table_name	comment
 18	disk_write_speed_base	Actual speed of disk writes per LDM thread, base data
 19	disk_write_speed_aggregate	Actual speed of disk writes per LDM thread, aggregate data
 20	frag_operations	Per fragment operational information
+21	restart_info	Times of restart phases in seconds and current state
 SELECT COUNT(*) FROM ndb$tables;
 COUNT(*)
-21
+22
 SELECT * FROM ndb$tables WHERE table_id = 2;
 table_id	table_name	comment
 2	test	for testing
@@ -88,11 +89,12 @@ table_id	table_name	comment
 18	disk_write_speed_base	Actual speed of disk writes per LDM thread, base data
 19	disk_write_speed_aggregate	Actual speed of disk writes per LDM thread, aggregate data
 20	frag_operations	Per fragment operational information
+21	restart_info	Times of restart phases in seconds and current state
 SELECT * FROM ndb$tables WHERE table_name = 'LOGDESTINATION';
 table_id	table_name	comment
 SELECT COUNT(*) FROM ndb$tables t1, ndb$tables t2 WHERE t1.table_id = t1.table_id;
 COUNT(*)
-441
+484
 
 SELECT table_id, table_name, comment from ndb$tables
   WHERE table_id > 2 AND table_id <= 5 ORDER BY table_id;
@@ -120,6 +122,7 @@ table_id	table_name
 14	operations
 3	pools
 7	resources
+21	restart_info
 0	tables
 2	test
 11	threadblocks
@@ -182,6 +185,7 @@ table_id
 18
 19
 20
+21
 
 TRUNCATE ndb$tables;
 ERROR HY000: Table 'ndb$tables' is read only
@@ -461,6 +465,33 @@ select count(*) from disk_write_speed_aggregate_node;
 count(*)
 2
 
+desc restart_info;
+Field	Type	Null	Key	Default	Extra
+node_id	int(10) unsigned	YES		NULL	
+node_restart_status	varchar(256)	YES		NULL	
+node_restart_status_int	int(10) unsigned	YES		NULL	
+secs_to_complete_node_failure	int(10) unsigned	YES		NULL	
+secs_to_allocate_node_id	int(10) unsigned	YES		NULL	
+secs_to_include_in_heartbeat_protocol	int(10) unsigned	YES		NULL	
+secs_until_wait_for_ndbcntr_master	int(10) unsigned	YES		NULL	
+secs_wait_for_ndbcntr_master	int(10) unsigned	YES		NULL	
+secs_to_get_start_permitted	int(10) unsigned	YES		NULL	
+secs_to_wait_for_lcp_for_copy_meta_data	int(10) unsigned	YES		NULL	
+secs_to_copy_meta_data	int(10) unsigned	YES		NULL	
+secs_to_include_node	int(10) unsigned	YES		NULL	
+secs_starting_node_to_request_local_recovery	int(10) unsigned	YES		NULL	
+secs_for_local_recovery	int(10) unsigned	YES		NULL	
+secs_restore_fragments	int(10) unsigned	YES		NULL	
+secs_undo_disk_data	int(10) unsigned	YES		NULL	
+secs_exec_redo_log	int(10) unsigned	YES		NULL	
+secs_index_rebuild	int(10) unsigned	YES		NULL	
+secs_to_synchronize_starting_node	int(10) unsigned	YES		NULL	
+secs_wait_lcp_for_restart	int(10) unsigned	YES		NULL	
+secs_wait_subscription_handover	int(10) unsigned	YES		NULL	
+total_restart_secs	int(10) unsigned	YES		NULL	
+select * from restart_info;
+node_id	node_restart_status	node_restart_status_int	secs_to_complete_node_failure	secs_to_allocate_node_id	secs_to_include_in_heartbeat_protocol	secs_until_wait_for_ndbcntr_master	secs_wait_for_ndbcntr_master	secs_to_get_start_permitted	secs_to_wait_for_lcp_for_copy_meta_data	secs_to_copy_meta_data	secs_to_include_node	secs_starting_node_to_request_local_recovery	secs_for_local_recovery	secs_restore_fragments	secs_undo_disk_data	secs_exec_redo_log	secs_index_rebuild	secs_to_synchronize_starting_node	secs_wait_lcp_for_restart	secs_wait_subscription_handover	total_restart_secs
+
 desc cluster_transactions;
 Field	Type	Null	Key	Default	Extra
 node_id	int(10) unsigned	YES		NULL	

mysql-test/suite/ndb/t/ndbinfo.test

@@ -232,6 +232,9 @@ select node_id from disk_write_speed_base group by node_id, thr_no;
 
 select count(*) from disk_write_speed_aggregate_node;
 
+desc restart_info;
+select * from restart_info;
+
 desc cluster_transactions;
 desc server_transactions;
 desc cluster_operations;

scripts/mysql_system_tables.sql

@@ -1803,6 +1803,11 @@ PREPARE stmt FROM @str;
 EXECUTE stmt;
 DROP PREPARE stmt;
 
+SET @str=IF(@have_ndbinfo,'DROP VIEW IF EXISTS `ndbinfo`.`restart_info`','SET @dummy = 0');
+PREPARE stmt FROM @str;
+EXECUTE stmt;
+DROP PREPARE stmt;
+
 SET @str=IF(@have_ndbinfo,'DROP VIEW IF EXISTS `ndbinfo`.`cluster_transactions`','SET @dummy = 0');
 PREPARE stmt FROM @str;
 EXECUTE stmt;
@@ -2039,6 +2044,17 @@ PREPARE stmt FROM @str;
 EXECUTE stmt;
 DROP PREPARE stmt;
 
+# ndbinfo.ndb$restart_info
+SET @str=IF(@have_ndbinfo,'DROP TABLE IF EXISTS `ndbinfo`.`ndb$restart_info`','SET @dummy = 0');
+PREPARE stmt FROM @str;
+EXECUTE stmt;
+DROP PREPARE stmt;
+
+SET @str=IF(@have_ndbinfo,'CREATE TABLE `ndbinfo`.`ndb$restart_info` (`node_id` INT UNSIGNED COMMENT "node id",`node_restart_status` VARCHAR(256) COMMENT "Current state of node recovery",`node_restart_status_int` INT UNSIGNED COMMENT "Current state of node recovery as number",`secs_to_complete_node_failure` INT UNSIGNED COMMENT "Seconds to complete node failure handling",`secs_to_allocate_node_id` INT UNSIGNED COMMENT "Seconds from node failure completion to allocation of node id", `secs_to_include_in_heartbeat_protocol` INT UNSIGNED COMMENT "Seonds from allocation of node id to inclusion in HB protocol",`secs_until_wait_for_ndbcntr_master` INT UNSIGNED COMMENT "Seconds from included in HB protocol until we wait for ndbcntr master",`secs_wait_for_ndbcntr_master` INT UNSIGNED COMMENT "Seconds we waited for being accepted by NDBCNTR master to start",`secs_to_get_start_permitted` INT UNSIGNED COMMENT "Seconds from permit by master until all nodes accepted our start",`secs_to_wait_for_lcp_for_copy_meta_data` INT UNSIGNED COMMENT "Seconds waiting for LCP completion before copying meta data",`secs_to_copy_meta_data` INT UNSIGNED COMMENT "Seconds to copy meta data to starting node from master",`secs_to_include_node` INT UNSIGNED COMMENT "Seconds to wait for GCP and inclusion of all nodes into protocols",`secs_starting_node_to_request_local_recovery` INT UNSIGNED COMMENT "Seconds for starting node to request local recovery",`secs_for_local_recovery` INT UNSIGNED COMMENT "Seconds for local recovery in starting node",`secs_restore_fragments` INT UNSIGNED COMMENT "Seconds to restore fragments from LCP files",`secs_undo_disk_data` INT UNSIGNED COMMENT "Seconds to execute UNDO log on disk data part of records",`secs_exec_redo_log` INT UNSIGNED COMMENT "Seconds to execute REDO log on all restored fragments",`secs_index_rebuild` INT UNSIGNED COMMENT "Seconds to rebuild indexes on restored fragments",`secs_to_synchronize_starting_node` INT UNSIGNED COMMENT "Seconds to synchronize starting node from live nodes",`secs_wait_lcp_for_restart` INT UNSIGNED COMMENT "Seconds to wait for LCP start and completion before restart is completed",`secs_wait_subscription_handover` INT UNSIGNED COMMENT "Seconds waiting for handover of replication subscriptions",`total_restart_secs` INT UNSIGNED COMMENT "Total number of seconds from node failure until node is started again") COMMENT="Times of restart phases in seconds and current state" ENGINE=NDBINFO','SET @dummy = 0');
+PREPARE stmt FROM @str;
+EXECUTE stmt;
+DROP PREPARE stmt;
+
 # ndbinfo.ndb$transactions
 SET @str=IF(@have_ndbinfo,'DROP TABLE IF EXISTS `ndbinfo`.`ndb$transactions`','SET @dummy = 0');
 PREPARE stmt FROM @str;
@@ -2313,6 +2329,12 @@ PREPARE stmt FROM @str;
 EXECUTE stmt;
 DROP PREPARE stmt;
 
+# ndbinfo.restart_info
+SET @str=IF(@have_ndbinfo,'CREATE OR REPLACE DEFINER=`root`@`localhost` SQL SECURITY INVOKER VIEW `ndbinfo`.`restart_info` AS SELECT * from `ndbinfo`.`ndb$restart_info`','SET @dummy = 0');
+PREPARE stmt FROM @str;
+EXECUTE stmt;
+DROP PREPARE stmt;
+
 # ndbinfo.cluster_transactions
 SET @str=IF(@have_ndbinfo,'CREATE OR REPLACE DEFINER=`root`@`localhost` SQL SECURITY INVOKER VIEW `ndbinfo`.`cluster_transactions` AS SELECT t.node_id, t.block_instance, t.transid0 + (t.transid1 << 32) as transid, s.state_friendly_name as state,  t.c_ops as count_operations,  t.outstanding as outstanding_operations,  t.timer as inactive_seconds,  (t.apiref & 65535) as client_node_id,  (t.apiref >> 16) as client_block_ref FROM `ndbinfo`.`ndb$transactions` t LEFT JOIN `ndbinfo`.`ndb$dbtc_apiconnect_state` s        ON s.state_int_value = t.state','SET @dummy = 0');
 PREPARE stmt FROM @str;

storage/ndb/include/kernel/GlobalSignalNumbers.h

@@ -24,7 +24,7 @@
  *
  * When adding a new signal, remember to update MAX_GSN and SignalNames.cpp
  */
-const GlobalSignalNumber MAX_GSN = 769;
+const GlobalSignalNumber MAX_GSN = 778;
 
 struct GsnName {
   GlobalSignalNumber gsn;
@@ -1093,6 +1093,13 @@ extern const GlobalSignalNumber NO_OF_SIGNAL_NAMES;
 
 #define GSN_NODE_STARTED_REP            769
 
-#define GSN_CHECK_NODE_INCLUDED_REQ     770
-#define GSN_CHECK_NODE_INCLUDED_CONF    771
+#define GSN_ALLOC_NODEID_REP            770
+#define GSN_INCL_NODE_HB_PROTOCOL_REP   771
+#define GSN_NDBCNTR_START_WAIT_REP      772
+#define GSN_NDBCNTR_STARTED_REP         773
+#define GSN_SUMA_HANDOVER_COMPLETE_REP  774
+#define GSN_END_TOREP                   775
+#define GSN_LOCAL_RECOVERY_COMP_REP     776
+#define GSN_CHECK_NODE_INCLUDED_REQ     777
+#define GSN_CHECK_NODE_INCLUDED_CONF    778
 #endif

storage/ndb/include/kernel/signaldata/AllocNodeId.hpp

@@ -1,5 +1,5 @@
 /*
-   Copyright (c) 2006, 2013, Oracle and/or its affiliates. All rights reserved.
+   Copyright (c) 2006, 2014, Oracle and/or its affiliates. All rights reserved.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -76,6 +76,16 @@ class AllocNodeIdRef {
   Uint32 masterRef;
 };
 
+class AllocNodeIdRep
+{
+private:
+  friend class Dbdih;
+  friend class Qmgr;
+  STATIC_CONST( SignalLength = 1);
+
+  Uint32 nodeId;
+};
+
 #undef JAM_FILE_ID
 
 #endif

storage/ndb/include/kernel/signaldata/DihContinueB.hpp

@@ -73,7 +73,6 @@ class DihContinueB {
     ZTO_START_FRAGMENTS = 44
     ,ZWAIT_OLD_SCAN = 46
     ,ZLCP_TRY_LOCK = 47
-    ,ZDELAY_RELEASE_FRAGMENT_INFO_MUTEX = 48
     ,ZTO_START_LOGGING = 49
     ,ZGET_TABINFO = 50
     ,ZGET_TABINFO_SEND = 51

storage/ndb/include/kernel/signaldata/TakeOver.hpp

@@ -191,6 +191,13 @@ struct EndToConf
   Uint32 startingNodeId;
 };
 
+struct EndToRep
+{
+public:
+  STATIC_CONST ( SignalLength = 1 );
+
+  Uint32 nodeId;
+};
 
 #undef JAM_FILE_ID
 

storage/ndb/include/ndb_version.h.in

@@ -873,6 +873,12 @@ ndbd_multi_tc_instance_takeover(Uint32 x)
  */
 #define NDBD_EXTRA_PARALLEL_FRAG_LCP NDB_MAKE_VERSION(7,4,1)
 
+/**
+ * Support for Node Recovery Status module, including sending
+ * END_TOREP.
+ */
+#define NDBD_NODE_RECOVERY_STATUS_VERSION NDB_MAKE_VERSION(7,4,3)
+
 /**
  * Support for removing EMPTY_LCP_REQ protocol in master takeover
  */

storage/ndb/src/common/debugger/signaldata/DihContinueB.cpp

@@ -195,10 +195,6 @@ printCONTINUEB_DBDIH(FILE * output, const Uint32 * theData,
     fprintf(output, " Lcp trylock: attempt %u\n",
             theData[1]);
     break;
-  case DihContinueB::ZDELAY_RELEASE_FRAGMENT_INFO_MUTEX:
-    fprintf(output, " Delay release fragment-info mutex (handle: 0x%x)",
-            theData[1]);
-    break;
   default:
     fprintf(output, " Default system error lab...\n");
     break;

storage/ndb/src/common/debugger/signaldata/SignalNames.cpp

@@ -809,6 +809,14 @@ const GsnName SignalNames [] = {
   ,{ GSN_BUILD_FK_IMPL_REF, "BUILD_FK_IMPL_REF" }
   ,{ GSN_BUILD_FK_IMPL_CONF, "BUILD_FK_IMPL_CONF" }
   ,{ GSN_NODE_STARTED_REP, "NODE_STARTED_REP" }
+  ,{ GSN_ALLOC_NODEID_REP, "ALLOC_NODEID_REP" }
+  ,{ GSN_INCL_NODE_HB_PROTOCOL_REP, "INCL_NODE_HB_PROTOCOL_REP" }
+  ,{ GSN_NDBCNTR_START_WAIT_REP, "NDBCNTR_START_WAIT_REP" }
+  ,{ GSN_NDBCNTR_STARTED_REP, "NDBCNTR_STARTED_REP" }
+  ,{ GSN_SUMA_HANDOVER_COMPLETE_REP, "SUMA_HANDOVER_COMPLETE_REP" }
+  ,{ GSN_END_TOREP, "END_TOREP" }
+  ,{ GSN_LOCAL_RECOVERY_COMP_REP, "LOCAL_RECOVERY_COMP_REP" }
+  ,{ GSN_CANCEL_SUBSCRIPTION_REQ, "CANCEL_SUBSCRIPTION_REQ" }
   ,{ GSN_CHECK_NODE_INCLUDED_REQ, "CHECK_NODE_INCLUDED_REQ" }
   ,{ GSN_CHECK_NODE_INCLUDED_CONF, "CHECK_NODE_INCLUDED_CONF" }
 };

storage/ndb/src/kernel/blocks/dbdict/Dbdict.cpp

@@ -33,6 +33,7 @@
 #include <AttributeHeader.hpp>
 #include <KeyDescriptor.hpp>
 #include <Checksum.hpp>
+#include <signaldata/NodeRecoveryStatusRep.hpp>
 #include <signaldata/DictSchemaInfo.hpp>
 #include <signaldata/DictTabInfo.hpp>
 #include <signaldata/DropTabFile.hpp>
@@ -20428,9 +20429,24 @@ Dbdict::execDICT_UNLOCK_ORD(Signal* signal)
   if (ord->lockType == DictLockReq::SumaStartMe ||
       ord->lockType == DictLockReq::SumaHandOver)
   {
+    Uint32 nodeId = refToNode(ord->senderRef);
     jam();
-    g_eventLogger->info("clearing SumaStartMe dict lock for %u", refToNode(ord->senderRef));
-    c_sub_startstop_lock.clear(refToNode(ord->senderRef));
+    g_eventLogger->info("clearing SumaStartMe dict lock for %u", nodeId);
+    c_sub_startstop_lock.clear(nodeId);
+
+    if (ord->lockType == DictLockReq::SumaHandOver)
+    {
+      /**
+       * Inform the master DIH that the SUMA handover is now completed, this
+       * is the very last phase of the node recovery. This code is only
+       * executed in the master node.
+       */
+      SumaHandoverCompleteRep *rep =
+        (SumaHandoverCompleteRep*)signal->getDataPtrSend();
+      rep->nodeId = nodeId;
+      EXECUTE_DIRECT(DBDIH, GSN_SUMA_HANDOVER_COMPLETE_REP, signal,
+                     SumaHandoverCompleteRep::SignalLength);
+    }
     return;
   }