bug#36694848 Configure ndb binlog cache size

Problem:
The ndbcluster plugin subscribes to all changes that occur in NDB and
writes them epoch by epoch to binary log. Each epoch received from NDB
might consist of a large number of changes which are all written to the
binlog transaction cache before flushing them to the binary log. In
order to properly dimension and configure the system the binlog
transaction cache used by the ndb binlog injector must be possible to
configure separately from sessions that normally write transactions of
smaller size. Currently it is only possible to configure the cache size
for all sessions which leads to improper resource allocation for a
MySQL Server used for writing binary log of changes in NDB.

Solution:
Make it possible to configure the binlog transaction cache size
used for ndb binlog injector with a new system variable:
  --ndb-log-cache-size=<value>
The new variable can be changed at runtime and it's default value
is set to 64MB. All in all the new variable is very much like --binlog-
cache-size, just that it only affects the transaction cache used
by ndb binlog injector when writing binary log of changes in NDB.

Change-Id: Ia6b7ddd238a132fdf74248a5616a06316197ba63

Author: blaudden

mysql-test/suite/ndb/r/ndb_basic.result

@@ -172,6 +172,7 @@ ndb_join_pushdown	#
 ndb_log_apply_status	#
 ndb_log_bin	#
 ndb_log_binlog_index	#
+ndb_log_cache_size	#
 ndb_log_empty_epochs	#
 ndb_log_empty_update	#
 ndb_log_exclusive_reads	#

mysql-test/suite/ndb/r/ndb_basic_3rpl.result

@@ -172,6 +172,7 @@ ndb_join_pushdown	#
 ndb_log_apply_status	#
 ndb_log_bin	#
 ndb_log_binlog_index	#
+ndb_log_cache_size	#
 ndb_log_empty_epochs	#
 ndb_log_empty_update	#
 ndb_log_exclusive_reads	#

mysql-test/suite/ndb/r/ndb_basic_4rpl.result

@@ -172,6 +172,7 @@ ndb_join_pushdown	#
 ndb_log_apply_status	#
 ndb_log_bin	#
 ndb_log_binlog_index	#
+ndb_log_cache_size	#
 ndb_log_empty_epochs	#
 ndb_log_empty_update	#
 ndb_log_exclusive_reads	#

mysql-test/suite/ndb/r/ndb_basic_ndbd.result

@@ -172,6 +172,7 @@ ndb_join_pushdown	#
 ndb_log_apply_status	#
 ndb_log_bin	#
 ndb_log_binlog_index	#
+ndb_log_cache_size	#
 ndb_log_empty_epochs	#
 ndb_log_empty_update	#
 ndb_log_exclusive_reads	#

mysql-test/suite/ndb/t/concurrent_grants.cnf

@@ -44,6 +44,8 @@ ndbcluster
 ndb-wait-connected=30
 ndb-wait-setup=120
 ndb-extra-logging=99
+# Use smaller than default cache size to save resources
+ndb-log-cache-size=32768
 
 [mysqld.1.1]
 ndb-cluster-connection-pool=1

mysql-test/suite/ndb/t/ndb_addnode_withbinlog.cnf

@@ -16,6 +16,8 @@ ndb-cluster-connection-pool=3
 ndb-extra-logging=99
 log-bin=mysql-binlog
 ndb-log-bin=1
+# Use smaller than default cache size to save resources
+ndb-log-cache-size=32768
 # Enable load data local functionality
 local-infile=true
 

mysql-test/suite/ndb_binlog/my.cnf

@@ -21,6 +21,8 @@ ndb-extra-logging=99
 local-infile=true
 
 ndb-log-bin=1
+# Use smaller than default cache size to save resources
+ndb-log-cache-size=32768
 
 # Run this suite with binlog-format=ROW by default
 binlog-format=                 row

mysql-test/suite/ndb_binlog/r/ndb_binlog_cache.result

@@ -1,6 +1,16 @@
+SET @save_ndb_log_cache_size = @@global.ndb_log_cache_size;
+SELECT @save_ndb_log_cache_size;
+@save_ndb_log_cache_size
+32768
+SET @@global.ndb_log_cache_size=4096;
+SET SESSION ndb_log_cache_size=32768;
+ERROR HY000: Variable 'ndb_log_cache_size' is a GLOBAL variable and should be set with SET GLOBAL
+SELECT @@global.ndb_log_cache_size;
+@@global.ndb_log_cache_size
+4096
 SELECT @@global.binlog_cache_size;
 @@global.binlog_cache_size
-4096
+32768
 SELECT @@global.binlog_stmt_cache_size;
 @@global.binlog_stmt_cache_size
 32768
@@ -36,5 +46,24 @@ INSERT INTO t1 VALUES (7, repeat(0x41, 4096), repeat(0x42, 4096));
 include/assert.inc [There were >0 writes to IO_CACHE]
 # Check that error log contains message indicating cache spill
 include/assert.inc [Binary log cache data overflowed to disk]
+include/assert.inc [The IO_CACHE should spill to disk]
+#
+# Check that ndb_log_cache_size can be reconfigured at runtime and the
+# large write which previously caused cache spill now fits. This is done
+# by increasing cache size and doing the large write again.
+SET @@global.ndb_log_cache_size=1024*1024;
+BEGIN;
+INSERT INTO t1 VALUES (8, repeat(0x41, 1024), repeat(0x42, 1024));
+INSERT INTO t1 VALUES (9, repeat(0x41, 1024), repeat(0x42, 1024));
+INSERT INTO t1 VALUES (10, repeat(0x41, 1024), repeat(0x42, 1024));
+INSERT INTO t1 VALUES (11, repeat(0x41, 1024), repeat(0x42, 1024));
+INSERT INTO t1 VALUES (12, repeat(0x41, 1024), repeat(0x42, 1024));
+INSERT INTO t1 VALUES (13, repeat(0x41, 1024), repeat(0x42, 1024));
+INSERT INTO t1 VALUES (14, repeat(0x41, 1024), repeat(0x42, 1024));
+INSERT INTO t1 VALUES (15, repeat(0x41, 1024), repeat(0x42, 1024));
+COMMIT;
+# Wait for ndb_binlog thread...
+include/assert.inc [Write with large cache didn't trigger cache overflow]
+SET @@global.ndb_log_cache_size=@save_ndb_log_cache_size;
 
 DROP TABLE t1;

mysql-test/suite/ndb_binlog/t/ndb_binlog_cache.cnf

@@ -6,8 +6,22 @@
 -- source include/have_ndb.inc
 -- source include/have_binlog_format_mixed_or_row.inc
 
+# Save original value to allow restoring config before test end
+SET @save_ndb_log_cache_size = @@global.ndb_log_cache_size;
+SELECT @save_ndb_log_cache_size;
+
+# Configure lowest possible ndb binlog injector cache size to make it possible
+# to exceed the size and trigger a "disk spill" further down
+SET @@global.ndb_log_cache_size=4096;
+
+# Show that ndb_log_cache_size is a global variable
+--error ER_GLOBAL_VARIABLE
+SET SESSION ndb_log_cache_size=32768;
+
 -- let $cache_qry = SELECT COUNT_WRITE FROM performance_schema.file_summary_by_event_name WHERE event_name LIKE "%io_cache%"
 
+# Check configured values required by test
+SELECT @@global.ndb_log_cache_size;
 SELECT @@global.binlog_cache_size;
 SELECT @@global.binlog_stmt_cache_size;
 SELECT NAME, ENABLED FROM performance_schema.setup_instruments WHERE name LIKE "%file/sql/io_cache%";
@@ -69,10 +83,50 @@ save_master_pos;
 -- let $assert_cond= [$qry, warnings, 1] >= 1
 -- source include/assert.inc
 
+# The temporary file used for extending  the cache in the above case can be
+# seen in performance_schema by quering the file_instances table, the file
+# has "ML" as file name prefix:
+#   SELECT * FROM performance_schema.file_instances
+#     WHERE EVENT_NAME = "wait/io/file/sql/io_cache";
+#   FILE_NAME                EVENT_NAME                 OPEN_COUNT
+#   /tmp/mysqld.1.1/MLfd=66  wait/io/file/sql/io_cache    1
+--let $assert_cond = OPEN_COUNT >= 1 FROM performance_schema.file_instances WHERE EVENT_NAME = "wait/io/file/sql/io_cache"
+--let $assert_text = The IO_CACHE should spill to disk
+--source include/assert.inc
+
 # Supress the provoked message
 -- disable_query_log ONCE
 call mtr.add_suppression(".*Binary log cache data overflowed to disk.*");
 
+--echo #
+--echo # Check that ndb_log_cache_size can be reconfigured at runtime and the
+--echo # large write which previously caused cache spill now fits. This is done
+--echo # by increasing cache size and doing the large write again.
+
+SET @@global.ndb_log_cache_size=1024*1024;
+BEGIN;
+INSERT INTO t1 VALUES (8, repeat(0x41, 1024), repeat(0x42, 1024));
+INSERT INTO t1 VALUES (9, repeat(0x41, 1024), repeat(0x42, 1024));
+INSERT INTO t1 VALUES (10, repeat(0x41, 1024), repeat(0x42, 1024));
+INSERT INTO t1 VALUES (11, repeat(0x41, 1024), repeat(0x42, 1024));
+INSERT INTO t1 VALUES (12, repeat(0x41, 1024), repeat(0x42, 1024));
+INSERT INTO t1 VALUES (13, repeat(0x41, 1024), repeat(0x42, 1024));
+INSERT INTO t1 VALUES (14, repeat(0x41, 1024), repeat(0x42, 1024));
+INSERT INTO t1 VALUES (15, repeat(0x41, 1024), repeat(0x42, 1024));
+COMMIT;
+--source suite/ndb/include/ndb_binlog_wait_own_changes.inc
+
+-- let $after_writes_with_large_cache = `$cache_qry`
+#echo after_writes_with_large_cache:  $after_writes_with_large_cache;
+
+-- let $assert_text= Write with large cache didn't trigger cache overflow
+-- let $assert_cond= $after_writes = $after_writes_with_large_cache
+-- source include/assert.inc
+
+# Restore changed config to original value
+SET @@global.ndb_log_cache_size=@save_ndb_log_cache_size;
+
+
 -- echo
 
 DROP TABLE t1;

mysql-test/suite/ndb_binlog/t/ndb_binlog_cache.test

@@ -20,6 +20,8 @@ ndb-wait-setup=120
 ndb-cluster-connection-pool=3
 ndb-extra-logging=99
 ndb-log-bin=1
+# Use smaller than default cache size to save resources
+ndb-log-cache-size=32768
 
 # Will be "auto upgraded" to MIXED
 binlog-format=STATEMENT

mysql-test/suite/ndb_binlog/t/ndb_binlog_default_format.cnf

@@ -19,7 +19,8 @@ ndb-wait-setup=120
 ndb-cluster-connection-pool=1
 ndb-log-bin=1
 binlog-format=row
-
+# Use smaller than default cache size to save resources
+ndb-log-cache-size=32768
 ndb-log-empty-update
 
 # Enable load data local functionality

mysql-test/suite/ndb_binlog/t/ndb_binlog_log_reads.cnf

@@ -19,6 +19,8 @@ ndb-wait-setup=120
 ndb-cluster-connection-pool=1
 ndb-log-bin=1
 binlog-format=row
+# Use smaller than default cache size to save resources
+ndb-log-cache-size=32768
 
 [cluster_config.mysqld.1.1]
 NodeId=49

mysql-test/suite/ndb_binlog/t/ndb_binlog_variant_ddl.cnf

@@ -19,6 +19,8 @@ ndb-wait-setup=120
 ndb-cluster-connection-pool=1
 ndb-log-bin=1
 binlog-format=row
+# Use smaller than default cache size to save resources
+ndb-log-cache-size=32768
 
 # Enable load data local functionality
 local-infile=true

mysql-test/suite/ndb_binlog/t/ndb_binlog_variants.cnf

@@ -2,3 +2,5 @@
 
 [mysqld]
 ndb-log-bin=1
+# Use smaller than default cache size to save resources
+ndb-log-cache-size=32768

mysql-test/suite/ndb_ddl/util_tables_drop.cnf

@@ -26,6 +26,8 @@ ndb-wait-setup=120
 ndb-cluster-connection-pool=3
 replica-allow-batching
 ndb-log-bin=1
+# Use smaller than default cache size to save resources
+ndb-log-cache-size=32768
 
 # Enable load data local functionality
 local-infile=true

mysql-test/suite/ndb_rpl/my.cnf

@@ -38,6 +38,8 @@ ndb-table-temporary=1
 ndb-extra-logging=99
 binlog-format=row
 replica-parallel-workers=1
+# Use smaller than default cache size to save resources
+ndb-log-cache-size=32768
 
 [cluster_config.mysqld.1.cluster1]
 NodeId=49

mysql-test/suite/ndb_rpl/t/ndb_rpl_3_cluster.cnf

@@ -35,6 +35,8 @@ log-replica-updates= OFF
 log-error-verbosity= 3
 binlog-format= row
 replica-parallel-workers= 1
+# Use smaller than default cache size to save resources
+ndb-log-cache-size=32768
 
 [cluster_config.mysqld.1.cluster1]
 NodeId= 10

mysql-test/suite/ndb_rpl/t/ndb_rpl_3site_no_log_updates.cnf

@@ -38,6 +38,8 @@ ndb-extra-logging=99
 log-error-verbosity=3
 binlog-format=row
 replica-parallel-workers=1
+# Use smaller than default cache size to save resources
+ndb-log-cache-size=32768
 
 [cluster_config.mysqld.1.cluster1]
 NodeId=10

mysql-test/suite/ndb_rpl/t/ndb_rpl_log_updates.cnf

@@ -29,6 +29,8 @@ ndb-wait-setup=120
 # Default log-replica-updates to off and turn it on
 # only selectively for some of the servers in config
 log-replica-updates=0
+# Use smaller than default cache size to save resources
+ndb-log-cache-size=32768
 
 
 [cluster_config.mysqld.1.1]

mysql-test/suite/ndb_rpl/t/ndb_rpl_multi_binlog_update.cnf

@@ -24,6 +24,8 @@ ndb-log-orig
 ndb-log-bin=1
 replica-parallel-workers=1
 replicate-ignore-table=test.abc
+# Use smaller than default cache size to save resources
+ndb-log-cache-size=32768
 
 [mysqld.1.cluster1]
 server_id=            1

mysql-test/suite/ndb_rpl/t/ndb_rpl_stored_grants.cnf

@@ -172,6 +172,7 @@ ndb_join_pushdown	#
 ndb_log_apply_status	#
 ndb_log_bin	#
 ndb_log_binlog_index	#
+ndb_log_cache_size	#
 ndb_log_empty_epochs	#
 ndb_log_empty_update	#
 ndb_log_exclusive_reads	#

mysql-test/suite/ndb_tls/ndb_basic.result

@@ -31,6 +31,8 @@ log-bin=                       master-bin
 # for performance reasons
 ndb-table-temporary=1
 ndb-log-bin=1
+# Use smaller than default cache size to save resources
+ndb-log-cache-size=32768
 replica-parallel-workers=1
 
 ndb-extra-logging=99

mysql-test/suite/rpl_ndb/my.cnf

@@ -8386,6 +8386,22 @@ int THD::binlog_setup_trx_data() {
   return 0;
 }
 
+bool THD::binlog_configure_trx_cache_size(ulong new_size) {
+  // Check expected block size.
+  assert((new_size % IO_SIZE) == 0);
+
+  binlog_cache_mngr *const cache_mngr = thd_get_cache_mngr(this);
+  if (cache_mngr == nullptr || !cache_mngr->is_binlog_empty()) {
+    // Must exist and be empty
+    return true;
+  }
+
+  // Close and reopen with new value
+  Binlog_cache_storage *const cache = cache_mngr->get_trx_cache();
+  cache->close();
+  return cache->open(new_size, max_binlog_cache_size);
+}
+
 /**
 
 */

sql/binlog.cc

@@ -1721,6 +1721,24 @@ class THD : public MDL_context_owner,
 
   int binlog_setup_trx_data();
 
+  /**
+   * @brief Configure size of binlog transaction cache. Used to configure the
+   * size of an individual cache, normally to a value that differs from the
+   * default `binlog_cache_size` which controls the size otherwise.
+   *
+   * @note Assumes that the binlog cache manager already exist (i.e created
+   * by call to binlog_setup_trx_data()) and is empty.
+   *
+   * @param new_size The new size of cache. Value exceeding
+   * `max_binlog_cache_size` will be clamped and warning logged. Value must be
+   * a multiple of IO_SIZE which is the block size for all binlog cache size
+   * related variables.
+   *
+   * @return true if new cache size can't be configured, in that case the cache
+   * is not usable.
+   */
+  bool binlog_configure_trx_cache_size(ulong new_size);
+
   /*
     Public interface to write RBR events to the binlog
   */

sql/sql_class.h

@@ -18186,6 +18186,37 @@ static MYSQL_SYSVAR_ULONG(
     0            /* block */
 );
 
+// Overrides --binlog-cache-size for the ndb binlog thread
+ulong opt_ndb_log_cache_size;
+static void fix_ndb_log_cache_size(THD *thd, SYS_VAR *, void *val_ptr,
+                                   const void *checked) {
+  ulong new_size = *static_cast<const ulong *>(checked);
+
+  // Cap the max value in the same way as other binlog cache size variables
+  if (new_size > max_binlog_cache_size) {
+    push_warning_printf(
+        thd, Sql_condition::SL_WARNING, ER_BINLOG_CACHE_SIZE_GREATER_THAN_MAX,
+        "Option ndb_log_cache_size (%lu) is greater than max_binlog_cache_size "
+        "(%lu); setting ndb_log_cache_size equal to max_binlog_cache_size.",
+        (ulong)new_size, (ulong)max_binlog_cache_size);
+    new_size = static_cast<ulong>(max_binlog_cache_size);
+  }
+  *(static_cast<ulong *>(val_ptr)) = new_size;
+}
+
+static MYSQL_SYSVAR_ULONG(
+    log_cache_size,         /* name */
+    opt_ndb_log_cache_size, /* var */
+    PLUGIN_VAR_RQCMDARG,
+    "Size of the binary log transaction cache used by NDB binlog",
+    nullptr,                /* check func. */
+    fix_ndb_log_cache_size, /* update func. */
+    64 * 1024 * 1024,       /* default */
+    IO_SIZE,                /* min */
+    ULONG_MAX,              /* max */
+    IO_SIZE                 /* block */
+);
+
 bool opt_ndb_clear_apply_status;
 static MYSQL_SYSVAR_BOOL(
     clear_apply_status,         /* name */
@@ -18599,6 +18630,7 @@ static SYS_VAR *system_variables[] = {
     MYSQL_SYSVAR(log_transaction_compression),
     MYSQL_SYSVAR(log_transaction_compression_level_zstd),
     MYSQL_SYSVAR(log_purge_rate),
+    MYSQL_SYSVAR(log_cache_size),
     MYSQL_SYSVAR(log_fail_terminate),
     MYSQL_SYSVAR(log_transaction_dependency),
     MYSQL_SYSVAR(clear_apply_status),