Bug #30215068 BINLOG ROTATION DEADLOCK WHEN INNODB CONCURRENCY LIMIT SETTED

PROBLEM
-------

1. In SBR, replication related mutexes are acquired while rotating a binlog. These
   mutexes are released only after updating a gtid table.
2. Before the rotate thread can commit ,many update transactions which
   are not updating any rows are fired which enter innodb and
   increase the srv_conc.n_active connection. Since no rows are
   updated it will not call MYSQL_BIN_LOG::prepare() which would
   have decremnted the srv_conc.n_active. These update threads later
   wait on the replication mutexes held by the rotate thread.
3. Since the innodb_thread_concurrency is set to very low value
   the rotate thread cannot enter innodb since innodb_thread_concurrency
   has been reached ,so there is a deadlock
4. This issue is not seen in RBR.

FIX
---

1. Consider rotate thread as a special thread which updates the gtid implicitly and
   and skip innodb thread concurrency check for it. The fix is backport
   from Bug #30427369.
2. Patch even skips threads having active attacable transactions. Even these
   can result in deadlock in low innodb_thread_concurrency setting.
   This fix is a backport of Bug#3109077.

Reviewed by : Praveenkumar Hulakund <praveenkumar.hulakund@oracle.com>
Reviewed by : Debarun Banerjee <debarun.banerjee@oracle.com>

Author: Aditya A

mysql-test/suite/binlog/r/binlog_rotate_deadlock.result

@@ -0,0 +1,26 @@
+#
+# Bug #30215068	BINLOG ROTATION DEADLOCK WHEN INNODB CONCURRENCY LIMIT SETTED
+#
+connect  con1, localhost, root,,;
+select  @@innodb_thread_concurrency;
+@@innodb_thread_concurrency
+1
+CREATE TABLE t1 (c1 INT) Engine=InnoDB;
+CREATE TABLE t2 (c1 INT, c2 int, c3 varchar(200)) Engine=InnoDB;
+SET @debug_save= @@SESSION.DEBUG;
+SET SESSION DEBUG = '+d,force_rotate';
+SET DEBUG_SYNC = 'stop_binlog_rotation_after_acquiring_lock_log SIGNAL rotate_stopped WAIT_FOR proceed_rotate';
+INSERT INTO t1 VALUES (1);;
+connect  con2, localhost, root,,;
+SET DEBUG_SYNC= 'now WAIT_FOR rotate_stopped';
+UPDATE t2 SET C2=1 WHERE C1 = 1;;
+connect  con3, localhost, root,,;
+SET DEBUG_SYNC= 'now SIGNAL proceed_rotate';
+connection con1;
+SET @@SESSION.DEBUG= @debug_save;
+connection con2;
+connection default;
+disconnect con1;
+disconnect con2;
+disconnect con3;
+DROP TABLE t1,t2;

mysql-test/suite/binlog/t/binlog_rotate_deadlock-master.opt

@@ -0,0 +1,7 @@
+--binlog-format=MIXED
+--max_binlog_size=4096
+--innodb_thread_concurrency=1
+--innodb_thread_sleep_delay=0
+--enforce_gtid_consistency=ON
+--gtid_mode=ON
+--master-info-repository=TABLE

mysql-test/suite/binlog/t/binlog_rotate_deadlock.test

@@ -0,0 +1,51 @@
+--source include/have_debug.inc
+--source include/have_debug_sync.inc
+--source include/have_binlog_format_statement.inc
+
+--echo #
+--echo # Bug #30215068	BINLOG ROTATION DEADLOCK WHEN INNODB CONCURRENCY LIMIT SETTED
+--echo #
+
+--enable_connect_log
+--connect (con1, localhost, root,,)
+select  @@innodb_thread_concurrency;
+
+CREATE TABLE t1 (c1 INT) Engine=InnoDB;
+CREATE TABLE t2 (c1 INT, c2 int, c3 varchar(200)) Engine=InnoDB;
+SET @debug_save= @@SESSION.DEBUG;
+SET SESSION DEBUG = '+d,force_rotate';
+SET DEBUG_SYNC = 'stop_binlog_rotation_after_acquiring_lock_log SIGNAL rotate_stopped WAIT_FOR proceed_rotate';
+--send INSERT INTO t1 VALUES (1);
+
+# Since innodb_thread_concurrency=1, Send txn from  a different connection
+
+--connect (con2, localhost, root,,)
+SET DEBUG_SYNC= 'now WAIT_FOR rotate_stopped';
+--SEND UPDATE t2 SET C2=1 WHERE C1 = 1;
+
+--connect (con3, localhost, root,,)
+# Verify that con1 is waiting in stop_binlog_rotation_after_acquiring_lock_log sync point and con 2 is waiting for con1 to commit
+--let $wait_condition=SELECT COUNT(*)=1 FROM INFORMATION_SCHEMA.PROCESSLIST WHERE State = 'debug sync point: stop_binlog_rotation_after_acquiring_lock_log' and info = 'INSERT INTO t1 VALUES (1)'
+--source include/wait_condition.inc
+--let $wait_condition=SELECT COUNT(*)=1 FROM INFORMATION_SCHEMA.PROCESSLIST WHERE State = 'query end' and info = 'UPDATE t2 SET C2=1 WHERE C1 = 1'
+--source include/wait_condition.inc
+
+# Continue rotate thread (con1)
+SET DEBUG_SYNC= 'now SIGNAL proceed_rotate';
+
+--connection con1
+#without fix con1 will hang because it cannot get innodb concurrency ticket
+--reap
+SET @@SESSION.DEBUG= @debug_save;
+
+--connection con2
+--reap
+
+# Cleanup
+connection default;
+
+--disconnect con1
+--disconnect con2
+--disconnect con3
+--disable_connect_log
+DROP TABLE t1,t2;

sql/binlog.cc

@@ -7580,6 +7580,8 @@ int MYSQL_BIN_LOG::rotate(bool force_rotate, bool* check_purge)
   assert(!is_relay_log);
   mysql_mutex_assert_owner(&LOCK_log);
 
+  DEBUG_SYNC(current_thd,"stop_binlog_rotation_after_acquiring_lock_log");
+
   *check_purge= false;
 
   if (DBUG_EVALUATE_IF("force_rotate", 1, 0) || force_rotate ||

sql/sql_class.cc

@@ -3897,6 +3897,16 @@ extern "C" int thd_non_transactional_update(const MYSQL_THD thd)
     Transaction_ctx::SESSION);
 }
 
+extern "C" int thd_has_active_attachable_trx(const MYSQL_THD thd)
+{
+  return thd->is_attachable_transaction_active();
+}
+
+extern "C" int thd_is_operating_gtid_table_implicitly(const MYSQL_THD thd)
+{
+  return thd->is_operating_gtid_table_implicitly;
+}
+
 extern "C" int thd_binlog_format(const MYSQL_THD thd)
 {
   if (mysql_bin_log.is_open() && (thd->variables.option_bits & OPTION_BIN_LOG))

sql/sql_class.h

@@ -3698,6 +3698,12 @@ class THD :public MDL_context_owner,
   /**
     @return true if there is an active attachable transaction.
   */
+  int is_attachable_transaction_active() const
+  { return m_attachable_trx != NULL; }
+
+  /**
+    @return true if there is an active attachable readonly transaction.
+  */
   bool is_attachable_ro_transaction_active() const
   { return m_attachable_trx != NULL && m_attachable_trx->is_read_only(); }
 

storage/innobase/handler/ha_innodb.cc

@@ -1507,13 +1507,12 @@ innobase_srv_conc_enter_innodb(
 	row_prebuilt_t*	prebuilt)
 {
 	/* We rely on server to do external_lock(F_UNLCK) to reset the
-	srv_conc.n_active counter. Since there are no locks on instrinsic
-	tables, we should skip this for intrinsic temporary tables. */
-	if (dict_table_is_intrinsic(prebuilt->table)) {
+	srv_conc.n_active counter.*/
+	if (skip_concurrency_ticket(prebuilt)) {
 		return;
 	}
 
-	trx_t*	trx	= prebuilt->trx;
+	trx_t*  trx     = prebuilt->trx;
 	if (srv_thread_concurrency) {
 		if (trx->n_tickets_to_enter_innodb > 0) {
 
@@ -1545,9 +1544,8 @@ innobase_srv_conc_exit_innodb(
 	row_prebuilt_t*	prebuilt)
 {
 	/* We rely on server to do external_lock(F_UNLCK) to reset the
-	srv_conc.n_active counter. Since there are no locks on instrinsic
-	tables, we should skip this for intrinsic temporary tables. */
-	if (dict_table_is_intrinsic(prebuilt->table)) {
+	srv_conc.n_active counter.*/
+	if (skip_concurrency_ticket(prebuilt)) {
 		return;
 	}
 
@@ -1586,6 +1584,36 @@ innobase_srv_conc_force_exit_innodb(
 	}
 }
 
+ibool
+skip_concurrency_ticket(row_prebuilt_t* prebuilt) {
+	/* Since there are no locks on instrinsic tables,
+	we should skip this for intrinsic temporary tables.*/
+	if (dict_table_is_intrinsic(prebuilt->table)) {
+		return true;
+	}
+
+	THD *thd =  prebuilt->trx->mysql_thd;
+	if (thd == NULL) {
+		thd = current_thd;
+	}
+	/* Skip concurrency ticket in following cases
+	 (a) while implicitly updating GTID table.This
+	 is to avoid deadlock otherwise possible with
+	 low innodb_thread_concurrency.
+	 Session: RESET MASTER -> FLUSH LOGS -> get innodb ticket
+	 -> wait for GTID flush.
+	 GTID Background: Write to GTID table -> wait for innodb ticket.
+         (b) For attachable transaction. If a attachable trx in
+	 thread asks for a ticket while the main transaction
+	 has reserved a ticket. */
+	if (thd && (thd_has_active_attachable_trx(thd)
+		|| thd_is_operating_gtid_table_implicitly(thd))) {
+		return true;
+	}
+
+	return false;
+}
+
 /******************************************************************//**
 Returns the NUL terminated value of glob_hostname.
 @return pointer to glob_hostname. */

storage/innobase/handler/ha_innodb.h

@@ -547,6 +547,18 @@ int thd_slave_thread(const MYSQL_THD thd);
 @retval 1 the user thread is running a non-transactional update */
 int thd_non_transactional_update(const MYSQL_THD thd);
 
+/** Check if the thread has attachable transaction
+@param thd user thread
+@retval 0 thd doesn't have attachable trx
+@retval 1 thd has attachable trx */
+int thd_has_active_attachable_trx(const MYSQL_THD thd);
+
+/** Check if the thread is doing a gtid operation implicitly
+@param thd user thread
+@retval 0 thd is not doing any gtid implicit operation
+@retval 1 thd is doing gtid implicit operation */
+int thd_is_operating_gtid_table_implicitly(const MYSQL_THD thd);
+
 /** Get the user thread's binary logging format
 @param thd user thread
 @return Value to be used as index into the binlog_format_names array */
@@ -600,6 +612,12 @@ typedef struct new_ft_info
 	fts_result_t*		ft_result;
 } NEW_FT_INFO;
 
+/* Check if the thread can skip  innodb concurrency check
+@param[in]	row_prebuilt_t*	 prebuilt
+@return true if thread can skip innodb concurrency check*/
+ibool
+skip_concurrency_ticket(row_prebuilt_t* prebuilt);
+
 /**
 Allocates an InnoDB transaction for a MySQL handler object.
 @return InnoDB transaction handle */