Skip to content

Commit b7a22c7

Browse files
author
Paweł Olchawa
committed
WL#12527 InnoDB: Dynamic configuration of space occupied by redo log files
Delivers an option to dynamically configure space occupied by redo log files during runtime. Introduces a new server variable, called "innodb_redo_log_capacity", which is a limitation for total size of all existing redo log files (expressed in bytes). Redo resize is performed online and its status can be tracked via new status variable: Innodb_redo_log_resize_status. Redo files are stored in #innodb_redo subdirectory and are named #ib_redoX, where X is growing and not wrapped. The innodb_log_file_size and innodb_log_files_in_group are deprecated. Existing redo files can be queried from innodb_redo_log_files PFS table. Redo log implementation is no longer based on the fil system. RB: 24307
1 parent 9bc47e4 commit b7a22c7

File tree

422 files changed

+20914
-8457
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

422 files changed

+20914
-8457
lines changed

include/my_dbug.h

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
/* Copyright (c) 2000, 2021, Oracle and/or its affiliates.
1+
/* Copyright (c) 2000, 2022, Oracle and/or its affiliates.
22
33
This program is free software; you can redistribute it and/or modify
44
it under the terms of the GNU General Public License, version 2.0,
@@ -313,6 +313,7 @@ extern void _db_flush_gcov_();
313313

314314
#define DBUG_LOG(keyword, v) \
315315
do { \
316+
_db_pargs_(__LINE__, keyword); \
316317
if (_db_enabled_()) { \
317318
std::ostringstream sout; \
318319
sout << v; \

mysql-test/collections/disabled.def

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,11 @@ innodb.log_first_rec_group : Bug#29692047 Disable for now
4141
innodb.fts_exec_interrupt : Bug#28534199 The test is unstable and should be rewritten
4242
innodb.slow_shutdown : Bug#31763837 mtr hits this debug assert post test run cleanup
4343
innodb.upgrade_fts_aux : Bug#33034679 Test needs an upgrade of its 5.7 data files
44+
innodb.instant_ddl_recovery_old : WL#12527 Redo format has changed - crashed_datadir.zip needs to be recreated.
45+
innodb.log_file_invalid_checkpoint : WL#12527 Test is shaky - needs to be fixed.
46+
innodb.log_logical_size : WL#12527 Test is shaky - needs to be fixed.
47+
innodb.log_file_invalid_start_lsn : WL#12527 Test is shaky - needs to be fixed.
48+
innodb.log_file_marked_as_full : WL#12527 Test is shaky - needs to be fixed.
4449

4550
# keyring tests
4651
keyring_encrypted_file.keyring_migration_bugs @darwin : Bug#31041633
Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
--disable_query_log
2+
CALL mtr.add_suppression("Skip re-populating collations and character sets tables in InnoDB read-only mode.");
3+
CALL mtr.add_suppression("Skip updating information_schema metadata in InnoDB read-only mode.");
4+
CALL mtr.add_suppression("Skipped updating resource group metadata in InnoDB read only mode.");
5+
--enable_query_log

mysql-test/include/default_mysqld.cnf

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,11 +13,11 @@ connect-timeout= 60
1313
log-bin-trust-function-creators=1
1414

1515
# Running MTR with --mem causes var directories to be created in memory.
16-
# Set innodb-buffer-pool-size and innodb-log-file-size to a lower
16+
# Set innodb-buffer-pool-size and innodb-redo-log-capacity to a lower
1717
# value because running tests in parallel, on the ramdisk may consume
1818
# excessive memory.
1919
innodb-buffer-pool-size=24M
20-
innodb-log-file-size=5M
20+
innodb-redo-log-capacity=10M
2121
# MAINTAINER:
2222
# the loose- syntax is to make sure the cnf file is also
2323
# valid when building without the performance schema.
Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
--disable_query_log
2+
DROP PROCEDURE log_spammer;
3+
DROP TABLE log_spam_controller;
4+
DROP TABLE log_spam;
5+
--enable_query_log
Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
--disable_query_log
2+
CREATE TABLE log_spam (a INT, b CHAR(100)) ENGINE=InnoDB;
3+
4+
CREATE TABLE log_spam_controller (
5+
id INT NOT NULL PRIMARY KEY,
6+
status ENUM('Active','Stopping','Stopped')) ENGINE=InnoDB;
7+
8+
INSERT INTO log_spam_controller (id, status) VALUES (1, 'Stopped');
9+
10+
DELIMITER |;
11+
CREATE PROCEDURE log_spam_init()
12+
BEGIN
13+
DECLARE i INT DEFAULT 1;
14+
WHILE (i <= 1000) DO
15+
INSERT INTO log_spam values (i, REPEAT('a',100));
16+
SET i = i + 1;
17+
END WHILE;
18+
END|
19+
20+
CREATE PROCEDURE log_spammer()
21+
BEGIN
22+
DECLARE i INT DEFAULT 1;
23+
DECLARE stop INT DEFAULT 0;
24+
UPDATE log_spam_controller SET status = 'Active' WHERE id = 1;
25+
WHILE (stop = 0) DO
26+
UPDATE log_spam SET b = RPAD(i, 100, 'x');
27+
SET i = i + 1;
28+
SELECT IF(status = 'Active',0,1) FROM log_spam_controller WHERE id = 1 INTO stop;
29+
END WHILE;
30+
UPDATE log_spam_controller SET status = 'Stopped' WHERE id = 1;
31+
END|
32+
DELIMITER ;|
33+
34+
CALL log_spam_init();
35+
DROP PROCEDURE log_spam_init;
36+
37+
--enable_query_log

mysql-test/include/ib_logfile_size_check.inc

Lines changed: 0 additions & 14 deletions
This file was deleted.
Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
# Checks if total size of all redo log files (including _tmp files) does not exceed
2+
# the provided physical capacity (MYSQLD_IB_REDO_LOG_PHYSICAL_CAPACITY). Expects to
3+
# find existing redo files (crashes if total size of redo files is zero).
4+
5+
perl;
6+
require 'include/ib_redo_log_files.inc';
7+
8+
my $physical_capacity = $ENV{'MYSQLD_IB_REDO_LOG_PHYSICAL_CAPACITY'};
9+
10+
print "Checking if total size of redo log files does not exceed $physical_capacity bytes\n";
11+
12+
my $size = 0;
13+
foreach (log_get_all_files()) {
14+
$size += -s $_;
15+
}
16+
$size <= $physical_capacity or die "Redo log files occupy too much space ($size > $physical_capacity).";
17+
$size > 0 or die "No redo log files have been found.";
18+
19+
EOF
Lines changed: 128 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,128 @@
1+
perl;
2+
use File::Spec;
3+
use List::Util qw[min max];
4+
5+
sub log_directory {
6+
return File::Spec->catdir($ENV{'MYSQLD_DATADIR'}, '#innodb_redo');
7+
}
8+
9+
sub log_get_all_files {
10+
my $log_dir = log_directory();
11+
return glob(File::Spec->catfile($log_dir, "#ib_redo*"));
12+
}
13+
14+
sub log_get_non_tmp_files {
15+
return grep(/#ib_redo\d+$/, log_get_all_files());
16+
}
17+
18+
sub log_get_newest_checkpoint {
19+
my $max_checkpoint_file = '';
20+
my $max_checkpoint_lsn = 0;
21+
foreach my $file_path (log_get_all_files()) {
22+
my $file_checkpoint_lsn = log_file_get_checkpoint_lsn($file_path);
23+
if ($file_checkpoint_lsn > $max_checkpoint_lsn) {
24+
$max_checkpoint_lsn = $file_checkpoint_lsn;
25+
$max_checkpoint_file = $file_path;
26+
}
27+
}
28+
return ($max_checkpoint_file, $max_checkpoint_lsn);
29+
}
30+
31+
sub log_file_get_header_size { return 2*1024; }
32+
33+
sub log_file_get_offset_for_lsn {
34+
my ($file_start_lsn, $lsn) = @_;
35+
return log_file_get_header_size() + ($lsn - $file_start_lsn);
36+
}
37+
38+
sub log_file_get_lsn_for_offset {
39+
my ($file_start_lsn, $offset) = @_;
40+
return $file_start_lsn + ($offset - log_file_get_header_size());
41+
}
42+
43+
sub log_file_get_block_no {
44+
my ($lsn) = @_;
45+
# Adding 1 because the same is done by log_block_convert_lsn_to_hdr_no() inside InnoDB
46+
# which this function pretends to mimic
47+
return 1 + int($lsn / 512);
48+
}
49+
50+
sub log_file_get_block_offset {
51+
my ($offset) = @_;
52+
return int($offset / 512) * 512;
53+
}
54+
55+
sub log_file_get_id {
56+
my ($file_path) = @_;
57+
$file_path =~ /#ib_redo(\d+)$/;
58+
return $1;
59+
}
60+
61+
sub log_file_get_checkpoint_lsn {
62+
my ($file_path) = @_;
63+
open my $fh, '<:raw', $file_path or die "open failed: $!\n";
64+
my $offset1 = 512 + 8;
65+
my $offset2 = 3*512 + 8;
66+
seek $fh, $offset1, 0;
67+
read $fh, my $packed_checkpoint_lsn1, 8 or die "read failed: $!\n";
68+
seek $fh, $offset2, 0;
69+
read $fh, my $packed_checkpoint_lsn2, 8 or die "read failed: $!\n";
70+
my $checkpoint_lsn1 = unpack('Q>', $packed_checkpoint_lsn1);
71+
my $checkpoint_lsn2 = unpack('Q>', $packed_checkpoint_lsn2);
72+
close $fh or die "close failed: $!\n";
73+
return max($checkpoint_lsn1, $checkpoint_lsn2);
74+
}
75+
76+
sub log_file_set_checkpoint_lsn {
77+
my ($file_path, $checkpoint_lsn1, $checkpoint_lsn2) = @_;
78+
open my $fh, '+<:raw', $file_path or die "open failed: $!\n";
79+
my $offset1 = 512 + 8;
80+
my $offset2 = 3*512 + 8;
81+
my $packed_checkpoint_lsn1 = pack('Q>', $checkpoint_lsn1);
82+
my $packed_checkpoint_lsn2 = pack('Q>', $checkpoint_lsn2);
83+
seek $fh, $offset1, 0;
84+
syswrite($fh, $packed_checkpoint_lsn1, 8) == 8 or die "write failed: $!\n";
85+
seek $fh, $offset2, 0;
86+
syswrite($fh, $packed_checkpoint_lsn2, 8) == 8 or die "write failed: $!\n";
87+
close $fh or die "close failed: $!\n";
88+
}
89+
90+
sub log_file_get_start_lsn {
91+
my ($file_path) = @_;
92+
open my $fh, '<:raw', $file_path or die "open failed: $!\n";
93+
seek $fh, 8, 0;
94+
read $fh, my $packed_start_lsn, 8 or die "read failed: $!\n";
95+
my $start_lsn = unpack('Q>', $packed_start_lsn);
96+
close $fh or die "close failed: $!\n";
97+
return $start_lsn;
98+
}
99+
100+
sub log_file_set_start_lsn {
101+
my ($file_path, $start_lsn) = @_;
102+
open my $fh, '+<:raw', $file_path or die "open failed: $!\n";
103+
seek $fh, 8, 0;
104+
my $packed_start_lsn = pack('Q>', $start_lsn);
105+
syswrite($fh, $packed_start_lsn, 8) == 8 or die "write failed: $!\n";
106+
close $fh or die "close failed: $!\n";
107+
}
108+
109+
sub log_file_get_flags {
110+
my ($file_path) = @_;
111+
open my $fh, '<:raw', $file_path or die "open of $file_path failed: $!\n";
112+
seek $fh, 48, 0;
113+
read $fh, my $packed_flags, 4 or die "read failed: $!\n";
114+
my $flags = unpack('L>', $packed_flags);
115+
close $fh or die "close failed: $!\n";
116+
return $flags;
117+
}
118+
119+
sub log_file_set_flags {
120+
my ($file_path, $flags) = @_;
121+
open my $fh, '+<:raw', $file_path or die "open of $file_path failed: $!\n";
122+
seek $fh, 48, 0;
123+
my $packed_flags = pack('L>', $flags);
124+
syswrite($fh, $packed_flags, 4) == 4 or die "write failed: $!\n";
125+
close $fh or die "close failed: $!\n";
126+
}
127+
128+
EOF
Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
perl;
2+
require 'include/ib_redo_log_files.inc';
3+
4+
my $non_tmp_min = $ENV{'MYSQLD_IB_REDO_LOG_NON_TMP_FILES_COUNT_MIN'};
5+
my $non_tmp_max = $ENV{'MYSQLD_IB_REDO_LOG_NON_TMP_FILES_COUNT_MAX'};
6+
7+
if ((defined $non_tmp_max) || (defined $non_tmp_min)) {
8+
while (true) {
9+
my $non_tmp_n = scalar(log_get_non_tmp_files());
10+
if ((defined $non_tmp_max) && $non_tmp_n <= $non_tmp_max) {
11+
print "There are $non_tmp_n / $non_tmp_max non-temporary redo log files.\n";
12+
last;
13+
}
14+
if ((defined $non_tmp_min) && $non_tmp_min <= $non_tmp_n) {
15+
print "There are $non_tmp_n >= $non_tmp_min non-temporary redo log files.\n";
16+
last;
17+
}
18+
19+
sleep 1;
20+
}
21+
}
22+
23+
my $all_max = $ENV{'MYSQLD_IB_REDO_LOG_ALL_FILES_COUNT_MAX'};
24+
if (defined $all_max) {
25+
while (true) {
26+
my $all_n = scalar(log_get_all_files());
27+
if ($all_n <= $all_max) {
28+
print "There are $all_n / $all_max redo log files.\n";
29+
last;
30+
}
31+
sleep 1;
32+
}
33+
}
34+
35+
EOF

mysql-test/include/keyring_tests/innodb/log_encrypt_2.inc

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33

44
#Suppress following messages from myslqd log
55
--disable_query_log
6-
call mtr.add_suppression("\\[ERROR\\] .*MY-\\d+.* Can't set redo log tablespace to be encrypted.");
6+
call mtr.add_suppression("\\[ERROR\\] .* Can't set redo log files to be encrypted.");
77
call mtr.add_suppression("You need to use --log-bin to make --binlog-format work");
88
--enable_query_log
99

mysql-test/include/keyring_tests/innodb/log_encrypt_3.inc

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ call mtr.add_suppression("Error while loading keyring content. The keyring might
77
call mtr.add_suppression("\\[Warning\\] .* Plugin mysqlx reported.*");
88
call mtr.add_suppression("\\[Warning\\] .* Ignoring tablespace .* because it could not be opened");
99
call mtr.add_suppression("\\[ERROR\\] .* Failed to find tablespace for table `\.\.*`\.`\.\.*` in the cache.");
10-
call mtr.add_suppression("\\[ERROR\\] .* Can't set redo log tablespace to be encrypted.");
10+
call mtr.add_suppression("\\[ERROR\\] .* Can't set redo log files to be encrypted.");
1111
--enable_query_log
1212

1313
# Restart with redo log encryption enabled

mysql-test/include/keyring_tests/innodb/log_encrypt_4.inc

Lines changed: 12 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -6,15 +6,17 @@
66
--disable_query_log
77
call mtr.add_suppression("Error while loading keyring content. The keyring might be malformed");
88
call mtr.add_suppression("Resizing redo log from");
9-
call mtr.add_suppression("Starting to delete and rewrite log files.");
10-
call mtr.add_suppression("New log files created, LSN=");
11-
call mtr.add_suppression("Can't set redo log tablespace to be encrypted.");
9+
call mtr.add_suppression("Redo log has been requested to resize");
10+
call mtr.add_suppression("Starting to delete and rewrite redo log files.");
11+
call mtr.add_suppression("New redo log files created, LSN=");
12+
call mtr.add_suppression("\\[ERROR\\] .* Can't set redo log files to be encrypted.");
1213
--enable_query_log
1314

1415
let $old_innodb_file_per_table = `SELECT @@innodb_file_per_table`;
1516
let $old_innodb_redo_log_encrypt = `SELECT @@innodb_redo_log_encrypt`;
1617
let $START_PAGE_SIZE= `select @@innodb_page_size`;
17-
let $LOG_FILE_SIZE= `select @@innodb_log_file_size`;
18+
let $LOG_SIZE= `select @@innodb_redo_log_capacity`;
19+
let $LOG_SIZE_SCALED_UP= `select @@innodb_redo_log_capacity * 3 div 2`;
1820
let $MYSQLD_BASEDIR= `select @@basedir`;
1921

2022
--mkdir $MYSQL_TMP_DIR/log_encrypt_dir1
@@ -34,13 +36,13 @@ EOF
3436
--echo # Stop the MTR default DB server
3537
--source include/shutdown_mysqld.inc
3638

37-
let NEW_CMD = $MYSQLD --no-defaults $PLUGIN_DIR_OPT --innodb_dedicated_server=OFF --initialize-insecure --innodb_log_file_size=$LOG_FILE_SIZE --innodb_page_size=$START_PAGE_SIZE --basedir=$MYSQLD_BASEDIR --datadir=$MYSQLD_DATADIR1 --init-file=$BOOTSTRAP_SQL --secure-file-priv="" --innodb_redo_log_encrypt=ON </dev/null>>$MYSQLTEST_VARDIR/tmp/bootstrap1.log 2>&1;
39+
let NEW_CMD = $MYSQLD --no-defaults $PLUGIN_DIR_OPT --innodb_dedicated_server=OFF --initialize-insecure --innodb_redo_log_capacity=$LOG_SIZE --innodb_page_size=$START_PAGE_SIZE --basedir=$MYSQLD_BASEDIR --datadir=$MYSQLD_DATADIR1 --init-file=$BOOTSTRAP_SQL --secure-file-priv="" --innodb_redo_log_encrypt=ON </dev/null>>$MYSQLTEST_VARDIR/tmp/bootstrap1.log 2>&1;
3840

3941
--echo # Run the bootstrap command of datadir1
4042
--exec $NEW_CMD
4143

4244
--echo # Start the DB server with datadir1
43-
--let $restart_parameters="restart: $PLUGIN_DIR_OPT --innodb_redo_log_encrypt=ON --datadir=$MYSQLD_DATADIR1 --innodb_page_size=$START_PAGE_SIZE --innodb_log_file_size=$LOG_FILE_SIZE"
45+
--let $restart_parameters="restart: $PLUGIN_DIR_OPT --innodb_redo_log_encrypt=ON --datadir=$MYSQLD_DATADIR1 --innodb_page_size=$START_PAGE_SIZE --innodb_redo_log_capacity=$LOG_SIZE"
4446
--replace_regex /\.dll/.so/
4547
--source include/start_mysqld_no_echo.inc
4648

@@ -80,13 +82,13 @@ let NEW_CMD = $MYSQLD --no-defaults $PLUGIN_DIR_OPT --innodb_dedicated_server=OF
8082
# Restore global manifest file
8183
--source include/keyring_tests/helper/binary_restore_manifest.inc
8284

83-
let NEW_CMD = $MYSQLD --no-defaults $PLUGIN_DIR_OPT --innodb_dedicated_server=OFF --initialize-insecure --innodb_log_file_size=$LOG_FILE_SIZE --innodb_page_size=$START_PAGE_SIZE --basedir=$MYSQLD_BASEDIR --datadir=$MYSQLD_DATADIR2 --init-file=$BOOTSTRAP_SQL --secure-file-priv="" --innodb_log_files_in_group=3 --innodb_redo_log_encrypt=ON </dev/null>>$MYSQLTEST_VARDIR/tmp/bootstrap2.log 2>&1;
85+
let NEW_CMD = $MYSQLD --no-defaults $PLUGIN_DIR_OPT --innodb_dedicated_server=OFF --initialize-insecure --innodb_redo_log_capacity=$LOG_SIZE_SCALED_UP --innodb_page_size=$START_PAGE_SIZE --basedir=$MYSQLD_BASEDIR --datadir=$MYSQLD_DATADIR2 --init-file=$BOOTSTRAP_SQL --secure-file-priv="" --innodb_redo_log_encrypt=ON </dev/null>>$MYSQLTEST_VARDIR/tmp/bootstrap2.log 2>&1;
8486

8587
--echo # Run the bootstrap command of datadir2
8688
--exec $NEW_CMD
8789

8890
--echo # Start the DB server with datadir2
89-
--let $restart_parameters="restart: $PLUGIN_DIR_OPT --innodb_redo_log_encrypt=ON --datadir=$MYSQLD_DATADIR2 --innodb_page_size=$START_PAGE_SIZE --innodb_log_file_size=$LOG_FILE_SIZE --innodb_log_files_in_group=3"
91+
--let $restart_parameters="restart: $PLUGIN_DIR_OPT --innodb_redo_log_encrypt=ON --datadir=$MYSQLD_DATADIR2 --innodb_page_size=$START_PAGE_SIZE --innodb_redo_log_capacity=$LOG_SIZE_SCALED_UP"
9092
--source include/start_mysqld_no_echo.inc
9193

9294
SELECT @@global.innodb_redo_log_encrypt ;
@@ -108,13 +110,13 @@ DROP TABLE tab1,tab2;
108110
--source include/keyring_tests/helper/local_keyring_file_remove.inc
109111
#------------------------------------------------------------------------------
110112

111-
let NEW_CMD = $MYSQLD --no-defaults $PLUGIN_DIR_OPT --innodb_dedicated_server=OFF --initialize-insecure --innodb_log_file_size=$LOG_FILE_SIZE --innodb_page_size=$START_PAGE_SIZE --basedir=$MYSQLD_BASEDIR --datadir=$MYSQLD_DATADIR3 --init-file=$BOOTSTRAP_SQL --secure-file-priv="" --innodb_redo_log_encrypt=ON </dev/null>>$MYSQLTEST_VARDIR/tmp/bootstrap3.log 2>&1;
113+
let NEW_CMD = $MYSQLD --no-defaults $PLUGIN_DIR_OPT --innodb_dedicated_server=OFF --initialize-insecure --innodb_redo_log_capacity=$LOG_SIZE --innodb_page_size=$START_PAGE_SIZE --basedir=$MYSQLD_BASEDIR --datadir=$MYSQLD_DATADIR3 --init-file=$BOOTSTRAP_SQL --secure-file-priv="" --innodb_redo_log_encrypt=ON </dev/null>>$MYSQLTEST_VARDIR/tmp/bootstrap3.log 2>&1;
112114

113115
--echo # Run the bootstrap command of datadir3
114116
--exec $NEW_CMD
115117

116118
--echo # Start the DB server with datadir3 and keyring loaded.
117-
--let $restart_parameters="restart: $PLUGIN_DIR_OPT --datadir=$MYSQLD_DATADIR3 --innodb_page_size=$START_PAGE_SIZE --innodb_log_file_size=$LOG_FILE_SIZE"
119+
--let $restart_parameters="restart: $PLUGIN_DIR_OPT --datadir=$MYSQLD_DATADIR3 --innodb_page_size=$START_PAGE_SIZE --innodb_redo_log_capacity=$LOG_SIZE"
118120
--source include/start_mysqld_no_echo.inc
119121

120122
SELECT @@global.innodb_redo_log_encrypt ;

0 commit comments

Comments
 (0)