Bug#23264552 -- XA: ASSERT `M_STATUS == DA_ERROR' IN MYSQL_ERRNO:SQL_ERROR.H:385

In case a storage engine failed to prepare a branch of an XA transaction
the method handlerton::prepare() returns nonzero value in the function ha_prepare().
To handle error condition the function ha_rollback_trans() is called to rollback
current transaction. While doing rollback the following sequence of functions is called
  ha_rollback_trans --> TC_LOG::rollback --> ha_rollback_low -->
    XID_STATE::set_error --> Diagnostics_area::mysql_errno.
It results in firing of the assert DBUG_ASSERT(m_status == DA_ERROR) since
no error has been set in Diagnostics_area. That leads to abnormal server
termination for the server built in DEBUG mode.

To fix it extra condition added in the function ha_rollback_low() to call
XID_STATE::set_error() only in case an error is set in Diagnostics_area.

(cherry picked from commit f28c4b9a41ec6929e15f174ef162a667c888f611)

Author: Dmitry Shulga

mysql-test/r/xa_debug.result

@@ -89,3 +89,18 @@ XA ROLLBACK 'xid1';
 ERROR XAE04: XAER_NOTA: Unknown XID
 XA START 'trx_another_one';
 DROP TABLE t1;
+#
+# Bug#23264552 - XA: ASSERT `m_status == da_error' IN mysql_errno:sql_error.h:385
+#
+CREATE TABLE t1 (a INT);
+XA START 'xid1';
+INSERT INTO t1 VALUES (1);
+XA END 'xid1';
+SET @@session.debug = '+d,simulate_xa_failure_prepare_in_engine';
+XA PREPARE 'xid1';
+ERROR XA100: XA_RBROLLBACK: Transaction branch was rolled back
+SET @@session.debug = '-d,simulate_xa_failure_prepare_in_engine';
+XA ROLLBACK 'xid1';
+ERROR XAE04: XAER_NOTA: Unknown XID
+XA START 'trx_another_one';
+DROP TABLE t1;

mysql-test/t/xa_debug.test

@@ -183,3 +183,31 @@ XA START 'trx_another_one';
 --disconnect con1
 --connection default
 DROP TABLE t1;
+
+--echo #
+--echo # Bug#23264552 - XA: ASSERT `m_status == da_error' IN mysql_errno:sql_error.h:385
+--echo #
+--connect (con1, 127.0.0.1,root,,test,$MASTER_MYPORT,)
+CREATE TABLE t1 (a INT);
+
+XA START 'xid1';
+INSERT INTO t1 VALUES (1);
+XA END 'xid1';
+SET @@session.debug = '+d,simulate_xa_failure_prepare_in_engine';
+--error ER_XA_RBROLLBACK
+XA PREPARE 'xid1';
+SET @@session.debug = '-d,simulate_xa_failure_prepare_in_engine';
+
+--error ER_XAER_NOTA
+XA ROLLBACK 'xid1';
+
+#
+# Check that subsequent XA transaction can be initiated.
+#
+
+XA START 'trx_another_one';
+--disconnect con1
+--source include/wait_until_disconnected.inc
+--connection default
+
+DROP TABLE t1;

sql/handler.cc

@@ -1986,8 +1986,22 @@ int ha_rollback_low(THD *thd, bool all)
   /*
     Thanks to possibility of MDL deadlock rollback request can come even if
     transaction hasn't been started in any transactional storage engine.
+
+    It is possible to have a call of ha_rollback_low() while handling
+    failure from ha_prepare() and an error in Daignostics_area still
+    wasn't set. Therefore it is required to check that an error in
+    Diagnostics_area is set before calling the method XID_STATE::set_error().
+
+    If it wasn't done it would lead to failure of the assertion
+      DBUG_ASSERT(m_status == DA_ERROR)
+    in the method Diagnostics_area::mysql_errno().
+
+    In case ha_prepare is failed and an error wasn't set in Diagnostics_area
+    the error ER_XA_RBROLLBACK is set in the Diagnostics_area from
+    the method Sql_cmd_xa_prepare::trans_xa_prepare() when non-zero result code
+    returned by ha_prepare() is handled.
   */
-  if (all && thd->transaction_rollback_request)
+  if (all && thd->transaction_rollback_request && thd->is_error())
     trn_ctx->xid_state()->set_error(thd);
 
   (void) RUN_HOOK(transaction, after_rollback, (thd, all));

sql/xa.cc

@@ -690,6 +690,12 @@ bool Sql_cmd_xa_prepare::trans_xa_prepare(THD *thd)
     DBUG_ASSERT(thd->m_transaction_psi == NULL);
 #endif
 
+    /*
+      Reset rm_error in case ha_prepare() returned error,
+      so thd->transaction.xid structure gets reset
+      by THD::transaction::cleanup().
+    */
+    thd->get_transaction()->xid_state()->reset_error();
     cleanup_trans_state(thd);
     xid_state->set_state(XID_STATE::XA_NOTR);
     thd->get_transaction()->cleanup();

storage/innobase/handler/ha_innodb.cc

@@ -16896,7 +16896,8 @@ innobase_xa_prepare(
 
 	TrxInInnoDB	trx_in_innodb(trx);
 
-	if (trx_in_innodb.is_aborted()) {
+	if (trx_in_innodb.is_aborted() ||
+	    DBUG_EVALUATE_IF("simulate_xa_failure_prepare_in_engine", 1, 0)) {
 
 		innobase_rollback(hton, thd, prepare_trx);