Skip to content

Commit 2e6e668

Browse files
author
Anibal Pinto
committed
WL#11123: Group Replication: hold reads and writes when the new primary has replication backlog to apply
This worklog implements a fencing mechanism when a new primary is being promoted in Group Replication (GR). The fencing will restrict connections from writing and reading from the new primary until it has applied all the pending backlog of changes that came from the old primary. Applications will not read stale data or do writes for a short period of time (during the new primary promotion).
1 parent bdfa6e0 commit 2e6e668

File tree

78 files changed

+3979
-1042
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

78 files changed

+3979
-1042
lines changed

‎include/mysql/plugin_group_replication.h

+9-2
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
/* Copyright (c) 2013, 2017, Oracle and/or its affiliates. All rights reserved.
1+
/* Copyright (c) 2013, 2018, Oracle and/or its affiliates. All rights reserved.
22
33
This program is free software; you can redistribute it and/or modify
44
it under the terms of the GNU General Public License, version 2.0,
@@ -29,7 +29,14 @@
2929
*/
3030

3131
#include <mysql/plugin.h>
32-
#define MYSQL_GROUP_REPLICATION_INTERFACE_VERSION 0x0102
32+
#define MYSQL_GROUP_REPLICATION_INTERFACE_VERSION 0x0103
33+
34+
enum enum_group_replication_consistency_level {
35+
// allow executing reads from newer primary even when backlog isn't applied
36+
GROUP_REPLICATION_CONSISTENCY_EVENTUAL,
37+
// hold data reads and writes on the new primary until applies all the backlog
38+
GROUP_REPLICATION_CONSISTENCY_BEFORE_ON_PRIMARY_FAILOVER
39+
};
3340

3441
/*
3542
Callbacks for get_connection_status_info function.

‎mysql-test/collections/default.push

+1-1
Original file line numberDiff line numberDiff line change
@@ -7,4 +7,4 @@
77
perl mysql-test-run.pl --timer --debug-server --force --parallel=auto --comment=all-default-debug --vardir=var-all-default --skip-combinations --unit-tests-report --no-skip
88

99
# Group Replication
10-
perl mysql-test-run.pl --timer --force --parallel=6 --comment=group_replication --vardir=var-group_replication --suite=group_replication
10+
perl mysql-test-run.pl --timer --debug-server --force --parallel=6 --comment=group_replication_parallel_applier-debug --vardir=var-group_replication_parallel_applier-debug --suite=group_replication --big-test --testcase-timeout=60 --suite-timeout=360 --mysqld=--slave-parallel-workers=4 --mysqld=--slave-parallel-type=logical_clock --mysqld=--slave_preserve_commit_order=ON

‎mysql-test/r/all_persisted_variables.result

+4-4
Original file line numberDiff line numberDiff line change
@@ -42,17 +42,17 @@ include/assert.inc ['Expect 554 variables in the table. Due to open Bugs, we are
4242

4343
# Test SET PERSIST
4444

45-
include/assert.inc [Expect 359 persisted variables in the table. Due to open Bugs, we are checking for 354]
45+
include/assert.inc [Expect 360 persisted variables in the table. Due to open Bugs, we are checking for 355]
4646

4747
************************************************************
4848
* 3. Restart server, it must preserve the persisted variable
4949
* settings. Verify persisted configuration.
5050
************************************************************
5151
# restart
5252

53-
include/assert.inc [Expect 354 persisted variables in persisted_variables table.]
54-
include/assert.inc [Expect 354 persisted variables shown as PERSISTED in variables_info table.]
55-
include/assert.inc [Expect 354 persisted variables with matching peristed and global values.]
53+
include/assert.inc [Expect 355 persisted variables in persisted_variables table.]
54+
include/assert.inc [Expect 355 persisted variables shown as PERSISTED in variables_info table.]
55+
include/assert.inc [Expect 355 persisted variables with matching peristed and global values.]
5656

5757
************************************************************
5858
* 4. Test RESET PERSIST IF EXISTS. Verify persisted variable

‎mysql-test/r/mysqld--help-notwin.result

+4
Original file line numberDiff line numberDiff line change
@@ -335,6 +335,9 @@ The following options may be given as the first argument:
335335
--group-concat-max-len=#
336336
The maximum length of the result of function
337337
GROUP_CONCAT()
338+
--group-replication-consistency[=name]
339+
Transaction consistency guarantee, possible values:
340+
EVENTUAL, BEFORE_ON_PRIMARY_FAILOVER
338341
--gtid-executed-compression-period[=#]
339342
When binlog is disabled, a background thread wakes up to
340343
compress the gtid_executed table every
@@ -1396,6 +1399,7 @@ ft-stopword-file (No default value)
13961399
gdb FALSE
13971400
general-log FALSE
13981401
group-concat-max-len 1024
1402+
group-replication-consistency EVENTUAL
13991403
gtid-executed-compression-period 1000
14001404
gtid-mode OFF
14011405
help TRUE

‎mysql-test/r/mysqld--help-win.result

+4
Original file line numberDiff line numberDiff line change
@@ -334,6 +334,9 @@ The following options may be given as the first argument:
334334
--group-concat-max-len=#
335335
The maximum length of the result of function
336336
GROUP_CONCAT()
337+
--group-replication-consistency[=name]
338+
Transaction consistency guarantee, possible values:
339+
EVENTUAL, BEFORE_ON_PRIMARY_FAILOVER
337340
--gtid-executed-compression-period[=#]
338341
When binlog is disabled, a background thread wakes up to
339342
compress the gtid_executed table every
@@ -1403,6 +1406,7 @@ ft-stopword-file (No default value)
14031406
gdb FALSE
14041407
general-log FALSE
14051408
group-concat-max-len 1024
1409+
group-replication-consistency EVENTUAL
14061410
gtid-executed-compression-period 1000
14071411
gtid-mode OFF
14081412
help TRUE

‎mysql-test/suite/group_replication/r/gr_group_consistency_privilege.result

+37
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
include/group_replication.inc
2+
Warnings:
3+
Note #### Sending passwords in plain text without SSL/TLS is extremely insecure.
4+
Note #### Storing MySQL user name or password information in the master info repository is not secure and is therefore not recommended. Please consider using the USER and PASSWORD connection options for START SLAVE; see the 'START SLAVE Syntax' in the MySQL Manual for more information.
5+
[connection server1]
6+
include/start_and_bootstrap_group_replication.inc
7+
8+
# 1. Create user with SYSTEM_VARIABLES_ADMIN privilege to allow change
9+
# globals variables
10+
CREATE USER 'group_rpl_user' IDENTIFIED BY '';
11+
GRANT SYSTEM_VARIABLES_ADMIN ON *.* TO 'group_rpl_user';
12+
13+
# 2. Connect to server using group_rpl_user
14+
[connection con_group_rpl_user]
15+
16+
# 3. Set group_replication_consistency at a global scope shall fail due
17+
# to lack of privileges
18+
SET @@GLOBAL.group_replication_consistency= BEFORE_ON_PRIMARY_FAILOVER;
19+
ERROR 42000: Access denied; you need (at least one of) the SUPER or GROUP_REPLICATION_ADMIN privilege(s) for this operation
20+
21+
# 4. Set group_replication_consistency at a session scope shall succeed,
22+
# do not need extra privileges
23+
SET @@SESSION.group_replication_consistency= BEFORE_ON_PRIMARY_FAILOVER;
24+
25+
# 5. Grant GROUP_REPLICATION_ADMIN privileges to group_rpl_user
26+
[connection default]
27+
GRANT GROUP_REPLICATION_ADMIN ON *.* TO 'group_rpl_user';
28+
29+
# 6. Set group_replication_consistency at a global scope shall succeed
30+
[connection con_group_rpl_user]
31+
SET @@GLOBAL.group_replication_consistency= BEFORE_ON_PRIMARY_FAILOVER;
32+
33+
# 7. Cleanup
34+
[connection default]
35+
SET @@GLOBAL.group_replication_consistency= DEFAULT;
36+
DROP USER group_rpl_user;
37+
include/group_replication_end.inc

‎mysql-test/suite/group_replication/r/gr_group_consistency_variable.result

+54
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
include/group_replication.inc
2+
Warnings:
3+
Note #### Sending passwords in plain text without SSL/TLS is extremely insecure.
4+
Note #### Storing MySQL user name or password information in the master info repository is not secure and is therefore not recommended. Please consider using the USER and PASSWORD connection options for START SLAVE; see the 'START SLAVE Syntax' in the MySQL Manual for more information.
5+
[connection server1]
6+
#
7+
# 1. Verify group_replication_consistency with invalid values: string,
8+
# out of range, decimal, and NULL. Expect ERROR.
9+
#
10+
SET GLOBAL group_replication_consistency= ON;
11+
ERROR 42000: Variable 'group_replication_consistency' can't be set to the value of 'ON'
12+
SET SESSION group_replication_consistency= OFF;
13+
ERROR 42000: Variable 'group_replication_consistency' can't be set to the value of 'OFF'
14+
SET GLOBAL group_replication_consistency= "";
15+
ERROR 42000: Variable 'group_replication_consistency' can't be set to the value of ''
16+
SET SESSION group_replication_consistency= "";
17+
ERROR 42000: Variable 'group_replication_consistency' can't be set to the value of ''
18+
SET GLOBAL group_replication_consistency= "B";
19+
ERROR 42000: Variable 'group_replication_consistency' can't be set to the value of 'B'
20+
SET SESSION group_replication_consistency= "E";
21+
ERROR 42000: Variable 'group_replication_consistency' can't be set to the value of 'E'
22+
SET GLOBAL group_replication_consistency= -1;
23+
ERROR 42000: Variable 'group_replication_consistency' can't be set to the value of '-1'
24+
SET SESSION group_replication_consistency= -1;
25+
ERROR 42000: Variable 'group_replication_consistency' can't be set to the value of '-1'
26+
SET GLOBAL group_replication_consistency= 5;
27+
ERROR 42000: Variable 'group_replication_consistency' can't be set to the value of '5'
28+
SET SESSION group_replication_consistency= 5;
29+
ERROR 42000: Variable 'group_replication_consistency' can't be set to the value of '5'
30+
SET GLOBAL group_replication_consistency= 1.0;
31+
ERROR 42000: Incorrect argument type to variable 'group_replication_consistency'
32+
SET SESSION group_replication_consistency= 1.0;
33+
ERROR 42000: Incorrect argument type to variable 'group_replication_consistency'
34+
SET GLOBAL group_replication_consistency= NULL;
35+
ERROR 42000: Variable 'group_replication_consistency' can't be set to the value of 'NULL'
36+
SET SESSION group_replication_consistency= NULL;
37+
ERROR 42000: Variable 'group_replication_consistency' can't be set to the value of 'NULL'
38+
#
39+
# 2. Verify group_replication_consistency with valid values.
40+
# Expect SUCCESS.
41+
SET GLOBAL group_replication_consistency= 1;
42+
SET SESSION group_replication_consistency= 1;
43+
SET GLOBAL group_replication_consistency= 0;
44+
SET SESSION group_replication_consistency= 0;
45+
SET GLOBAL group_replication_consistency= DEFAULT;
46+
SET SESSION group_replication_consistency= DEFAULT;
47+
SET GLOBAL group_replication_consistency= "BEFORE_ON_PRIMARY_FAILOVER";
48+
SET SESSION group_replication_consistency= "BEFORE_ON_PRIMARY_FAILOVER";
49+
SET GLOBAL group_replication_consistency= "EVENTUAL";
50+
SET SESSION group_replication_consistency= "EVENTUAL";
51+
#
52+
# 3. Clean up.
53+
#
54+
include/group_replication_end.inc

‎mysql-test/suite/group_replication/r/gr_persist_only_variables.result

+6-4
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ SET PERSIST_ONLY group_replication_bootstrap_group = @@GLOBAL.group_replication_
2626
SET PERSIST_ONLY group_replication_communication_debug_options = @@GLOBAL.group_replication_communication_debug_options;
2727
SET PERSIST_ONLY group_replication_components_stop_timeout = @@GLOBAL.group_replication_components_stop_timeout;
2828
SET PERSIST_ONLY group_replication_compression_threshold = @@GLOBAL.group_replication_compression_threshold;
29+
SET PERSIST_ONLY group_replication_consistency = @@GLOBAL.group_replication_consistency;
2930
SET PERSIST_ONLY group_replication_enforce_update_everywhere_checks = @@GLOBAL.group_replication_enforce_update_everywhere_checks;
3031
SET PERSIST_ONLY group_replication_exit_state_action = @@GLOBAL.group_replication_exit_state_action;
3132
SET PERSIST_ONLY group_replication_flow_control_applier_threshold = @@GLOBAL.group_replication_flow_control_applier_threshold;
@@ -67,7 +68,7 @@ SET PERSIST_ONLY group_replication_start_on_boot = @@GLOBAL.group_replication_st
6768
SET PERSIST_ONLY group_replication_transaction_size_limit = @@GLOBAL.group_replication_transaction_size_limit;
6869
SET PERSIST_ONLY group_replication_unreachable_majority_timeout = @@GLOBAL.group_replication_unreachable_majority_timeout;
6970

70-
include/assert.inc ['Expect 46 persisted variables.']
71+
include/assert.inc ['Expect 47 persisted variables.']
7172

7273
############################################################
7374
# 2. Restart server, it must bootstrap the group and preserve
@@ -76,9 +77,9 @@ include/assert.inc ['Expect 46 persisted variables.']
7677
include/rpl_reconnect.inc
7778
include/gr_wait_for_member_state.inc
7879

79-
include/assert.inc ['Expect 46 persisted variables in persisted_variables table.']
80-
include/assert.inc ['Expect 46 persisted variables shown as PERSISTED in variables_info table.']
81-
include/assert.inc ['Expect 38 persisted variables with matching persisted and global values.']
80+
include/assert.inc ['Expect 47 persisted variables in persisted_variables table.']
81+
include/assert.inc ['Expect 47 persisted variables shown as PERSISTED in variables_info table.']
82+
include/assert.inc ['Expect 39 persisted variables with matching persisted and global values.']
8283

8384
############################################################
8485
# 3. Test RESET PERSIST IF EXISTS.
@@ -89,6 +90,7 @@ RESET PERSIST IF EXISTS group_replication_bootstrap_group;
8990
RESET PERSIST IF EXISTS group_replication_communication_debug_options;
9091
RESET PERSIST IF EXISTS group_replication_components_stop_timeout;
9192
RESET PERSIST IF EXISTS group_replication_compression_threshold;
93+
RESET PERSIST IF EXISTS group_replication_consistency;
9294
RESET PERSIST IF EXISTS group_replication_enforce_update_everywhere_checks;
9395
RESET PERSIST IF EXISTS group_replication_exit_state_action;
9496
RESET PERSIST IF EXISTS group_replication_flow_control_applier_threshold;

‎mysql-test/suite/group_replication/r/gr_persist_variables.result

+6-4
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ SET PERSIST group_replication_bootstrap_group = @@GLOBAL.group_replication_boots
2626
SET PERSIST group_replication_communication_debug_options = @@GLOBAL.group_replication_communication_debug_options;
2727
SET PERSIST group_replication_components_stop_timeout = @@GLOBAL.group_replication_components_stop_timeout;
2828
SET PERSIST group_replication_compression_threshold = @@GLOBAL.group_replication_compression_threshold;
29+
SET PERSIST group_replication_consistency = @@GLOBAL.group_replication_consistency;
2930
SET PERSIST group_replication_enforce_update_everywhere_checks = @@GLOBAL.group_replication_enforce_update_everywhere_checks;
3031
SET PERSIST group_replication_exit_state_action = @@GLOBAL.group_replication_exit_state_action;
3132
SET PERSIST group_replication_flow_control_applier_threshold = @@GLOBAL.group_replication_flow_control_applier_threshold;
@@ -67,7 +68,7 @@ SET PERSIST group_replication_start_on_boot = @@GLOBAL.group_replication_start_o
6768
SET PERSIST group_replication_transaction_size_limit = @@GLOBAL.group_replication_transaction_size_limit;
6869
SET PERSIST group_replication_unreachable_majority_timeout = @@GLOBAL.group_replication_unreachable_majority_timeout;
6970

70-
include/assert.inc ['Expect 46 persisted variables.']
71+
include/assert.inc ['Expect 47 persisted variables.']
7172

7273
############################################################
7374
# 2. Restart server, it must bootstrap the group and preserve
@@ -76,9 +77,9 @@ include/assert.inc ['Expect 46 persisted variables.']
7677
include/rpl_reconnect.inc
7778
include/gr_wait_for_member_state.inc
7879

79-
include/assert.inc ['Expect 46 persisted variables in persisted_variables table.']
80-
include/assert.inc ['Expect 46 persisted variables shown as PERSISTED in variables_info table.']
81-
include/assert.inc ['Expect 46 persisted variables with matching persisted and global values.']
80+
include/assert.inc ['Expect 47 persisted variables in persisted_variables table.']
81+
include/assert.inc ['Expect 47 persisted variables shown as PERSISTED in variables_info table.']
82+
include/assert.inc ['Expect 47 persisted variables with matching persisted and global values.']
8283

8384
############################################################
8485
# 3. Test RESET PERSIST.
@@ -89,6 +90,7 @@ RESET PERSIST group_replication_bootstrap_group;
8990
RESET PERSIST group_replication_communication_debug_options;
9091
RESET PERSIST group_replication_components_stop_timeout;
9192
RESET PERSIST group_replication_compression_threshold;
93+
RESET PERSIST group_replication_consistency;
9294
RESET PERSIST group_replication_enforce_update_everywhere_checks;
9395
RESET PERSIST group_replication_exit_state_action;
9496
RESET PERSIST group_replication_flow_control_applier_threshold;

‎mysql-test/suite/group_replication/r/gr_primary_failover_consistency_cmd_not_hold.result

+77
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,77 @@
1+
include/group_replication.inc [rpl_server_count=2]
2+
Warnings:
3+
Note #### Sending passwords in plain text without SSL/TLS is extremely insecure.
4+
Note #### Storing MySQL user name or password information in the master info repository is not secure and is therefore not recommended. Please consider using the USER and PASSWORD connection options for START SLAVE; see the 'START SLAVE Syntax' in the MySQL Manual for more information.
5+
[connection server1]
6+
[connection server1]
7+
SET @@SESSION.group_replication_consistency= BEFORE_ON_PRIMARY_FAILOVER;
8+
include/start_and_bootstrap_group_replication.inc
9+
10+
# 1. Enable debug point that run an assert if command will be put on
11+
# hold
12+
[connection server1]
13+
SET @debug_save= @@GLOBAL.DEBUG;
14+
SET @@GLOBAL.DEBUG= '+d,launch_hook_trans_begin_assert_if_hold';
15+
16+
# 2. Validate SQL command SHOW will not block
17+
SHOW GLOBAL VARIABLES LIKE 'group_replication_consistency';
18+
Variable_name Value
19+
group_replication_consistency EVENTUAL
20+
21+
# 3. Validate SQL command SET will not block
22+
SET @@GLOBAL.group_replication_member_weight= DEFAULT;
23+
24+
# 4. Validate SQL command DO will not block
25+
DO SLEEP(1);
26+
27+
# 5. Validate SQL command EMPTY will not block
28+
29+
# 6. Validate SQL command USE will not block
30+
USE test;
31+
32+
# 7. Validate SQL command SELECT to performance_schema will not block
33+
SELECT COUNT(*) from performance_schema.replication_group_members;
34+
COUNT(*)
35+
1
36+
37+
# 8. Validate SQL command SELECT to table PROCESSLIST from database
38+
# infoschema will not block
39+
SELECT COUNT(*)=0 FROM INFORMATION_SCHEMA.PROCESSLIST WHERE STATE='debug sync point: now';
40+
COUNT(*)=0
41+
1
42+
43+
# 9. Validate SQL command SELECT from database syschema will not block
44+
SELECT COUNT(*)=1 FROM sys.version;
45+
COUNT(*)=1
46+
1
47+
48+
# 10. Validate SQL command SELECT that don't use tables will not block
49+
SELECT @@GLOBAL.GTID_EXECUTED;
50+
@@GLOBAL.GTID_EXECUTED
51+
GROUP_REPLICATION_GROUP_NAME:1
52+
53+
# 11. Validate SQL command RESET PERSIST will not block
54+
RESET PERSIST;
55+
56+
# 12. Validate SQL command STOP GROUP_REPLICATION will not block
57+
STOP GROUP_REPLICATION;
58+
59+
# 13. Reconnect server1 to group
60+
[connection server1]
61+
SET @@GLOBAL.DEBUG= '-d,launch_hook_trans_begin_assert_if_hold';
62+
include/start_and_bootstrap_group_replication.inc
63+
64+
# 14. Validate SQL command SHUTDOWN will not block
65+
SET @@GLOBAL.DEBUG= '+d,launch_hook_trans_begin_assert_if_hold';
66+
SHUTDOWN;
67+
include/rpl_reconnect.inc
68+
69+
# 15. Cleanup
70+
SET @@GLOBAL.DEBUG= '-d,launch_hook_trans_begin_assert_if_hold';
71+
SET GLOBAL group_replication_group_name= "GROUP_NAME";
72+
SET GLOBAL group_replication_local_address= "GROUP_LOCAL_ADDRESS2";
73+
SET GLOBAL group_replication_group_seeds= "GROUP_SEEDS_SERVER2";
74+
SET GLOBAL group_replication_enforce_update_everywhere_checks= FALSE;;
75+
SET GLOBAL group_replication_single_primary_mode= TRUE;;
76+
SET @@SESSION.group_replication_consistency= DEFAULT;
77+
include/group_replication_end.inc

0 commit comments

Comments
 (0)