Skip to content

Commit c581a65

Browse files
nacarvalhonacarvalho
committed
WL#12895: Group Replication: Add OFFLINE_MODE to group_replication_exit_state_action
The option group_replication_exit_state_action configures how Group Replication behaves when a server leaves the group unintentionally: 1. after encountering an applier error; 2. after encountering an recovery error; 3. in the case of a loss of majority; 4. when another member of the group expels it due to a suspicion timing out; 5. after an error on coordinated group changes[1]; 6. after a primary election error; 7. after a clone provisioning error; 8. when automatic rejoin is enable, after its attempts are exhausted unsuccessfully. This behavior can be: * READ_ONLY disable writes on the server, by enabling server super_read_only[2] option; * ABORT_SERVER shutdown the server. After this worklog is implemented, a new behavior is added: * OFFLINE_MODE close all connections and disallow new ones from users who do not have the CONNECTION_ADMIN or SUPER privilege, by enabling server offline_mode[3] option. OFFLINE_MODE includes the behaviour of READ_ONLY. This automatic connection closure will avoid user sessions to: 1) read stale data and 2) write data to orphans servers, without the need to shutdown the server. USER INTERFACE -------------- The DBA can specify the behaviour by setting the option: - name: group_replication_exit_state_action - values: { READ_ONLY, ABORT_SERVER, OFFLINE_MODE } - default: READ_ONLY - scope: global - dynamic: yes - replicated: no - persistable: PERSIST, PERSIST_ONLY - credentials: SYSTEM_VARIABLES_ADMIN - description: The action that is taken when the server leaves the group unintentionally. Possible values are READ_ONLY, ABORT_SERVER and OFFLINE_MODE. ReviewBoard: 21931
1 parent d546188 commit c581a65

File tree

61 files changed

+2702
-27
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

61 files changed

+2702
-27
lines changed

‎mysql-test/include/assert_and_disable_offline_mode.inc

+36
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
# ==== Purpose ====
2+
#
3+
# Assert that offline_mode variable is enabled and reset its value.
4+
#
5+
# ==== Usage ====
6+
#
7+
# [--let $skip_reset_offline_mode= 1]
8+
# [--let $rpl_debug= 1]
9+
# --source include/assert_and_disable_offline_mode.inc
10+
#
11+
# Parameters:
12+
# $skip_reset_offline_mode
13+
# Skip reset offline_mode variable.
14+
#
15+
# $rpl_debug
16+
# See include/rpl_init.inc
17+
18+
--let $include_filename= assert_and_disable_offline_mode.inc
19+
--source include/begin_include_file.inc
20+
21+
if (!$rpl_debug)
22+
{
23+
--disable_query_log
24+
}
25+
26+
--let $wait_condition= SELECT @@GLOBAL.offline_mode = 1
27+
--source include/wait_condition_or_abort.inc
28+
29+
if (!$skip_reset_offline_mode)
30+
{
31+
SET @@GLOBAL.offline_mode = OFF;
32+
}
33+
--let $skip_reset_offline_mode=
34+
35+
--let $include_filename= assert_and_disable_offline_mode.inc
36+
--source include/end_include_file.inc

‎mysql-test/r/grant_dynamic.result

+10
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ SELECT * FROM mysql.global_grants ORDER BY USER, PRIV, WITH_GRANT_OPTION;
1414
USER HOST PRIV WITH_GRANT_OPTION
1515
mysql.session localhost BACKUP_ADMIN N
1616
mysql.session localhost CLONE_ADMIN N
17+
mysql.session localhost CONNECTION_ADMIN N
1718
mysql.session localhost PERSIST_RO_VARIABLES_ADMIN N
1819
mysql.session localhost SESSION_VARIABLES_ADMIN N
1920
mysql.session localhost SYSTEM_USER N
@@ -52,6 +53,7 @@ SELECT * FROM mysql.global_grants ORDER BY USER, PRIV, WITH_GRANT_OPTION;
5253
USER HOST PRIV WITH_GRANT_OPTION
5354
mysql.session localhost BACKUP_ADMIN N
5455
mysql.session localhost CLONE_ADMIN N
56+
mysql.session localhost CONNECTION_ADMIN N
5557
mysql.session localhost PERSIST_RO_VARIABLES_ADMIN N
5658
mysql.session localhost SESSION_VARIABLES_ADMIN N
5759
mysql.session localhost SYSTEM_USER N
@@ -89,6 +91,7 @@ SELECT * FROM mysql.global_grants ORDER BY USER, PRIV, WITH_GRANT_OPTION;
8991
USER HOST PRIV WITH_GRANT_OPTION
9092
mysql.session localhost BACKUP_ADMIN N
9193
mysql.session localhost CLONE_ADMIN N
94+
mysql.session localhost CONNECTION_ADMIN N
9295
mysql.session localhost PERSIST_RO_VARIABLES_ADMIN N
9396
mysql.session localhost SESSION_VARIABLES_ADMIN N
9497
mysql.session localhost SYSTEM_USER N
@@ -139,6 +142,7 @@ SELECT * FROM mysql.global_grants ORDER BY USER, PRIV, WITH_GRANT_OPTION;
139142
USER HOST PRIV WITH_GRANT_OPTION
140143
mysql.session localhost BACKUP_ADMIN N
141144
mysql.session localhost CLONE_ADMIN N
145+
mysql.session localhost CONNECTION_ADMIN N
142146
mysql.session localhost PERSIST_RO_VARIABLES_ADMIN N
143147
mysql.session localhost SESSION_VARIABLES_ADMIN N
144148
mysql.session localhost SYSTEM_USER N
@@ -181,6 +185,7 @@ SELECT * FROM mysql.global_grants ORDER BY USER, PRIV, WITH_GRANT_OPTION;
181185
USER HOST PRIV WITH_GRANT_OPTION
182186
mysql.session localhost BACKUP_ADMIN N
183187
mysql.session localhost CLONE_ADMIN N
188+
mysql.session localhost CONNECTION_ADMIN N
184189
mysql.session localhost PERSIST_RO_VARIABLES_ADMIN N
185190
mysql.session localhost SESSION_VARIABLES_ADMIN N
186191
mysql.session localhost SYSTEM_USER N
@@ -218,6 +223,7 @@ SELECT * FROM mysql.global_grants ORDER BY USER, PRIV, WITH_GRANT_OPTION;
218223
USER HOST PRIV WITH_GRANT_OPTION
219224
mysql.session localhost BACKUP_ADMIN N
220225
mysql.session localhost CLONE_ADMIN N
226+
mysql.session localhost CONNECTION_ADMIN N
221227
mysql.session localhost PERSIST_RO_VARIABLES_ADMIN N
222228
mysql.session localhost SESSION_VARIABLES_ADMIN N
223229
mysql.session localhost SYSTEM_USER N
@@ -278,6 +284,7 @@ SELECT * FROM mysql.global_grants ORDER BY USER, PRIV, WITH_GRANT_OPTION;
278284
USER HOST PRIV WITH_GRANT_OPTION
279285
mysql.session localhost BACKUP_ADMIN N
280286
mysql.session localhost CLONE_ADMIN N
287+
mysql.session localhost CONNECTION_ADMIN N
281288
mysql.session localhost PERSIST_RO_VARIABLES_ADMIN N
282289
mysql.session localhost SESSION_VARIABLES_ADMIN N
283290
mysql.session localhost SYSTEM_USER N
@@ -329,6 +336,7 @@ SELECT * FROM mysql.global_grants ORDER BY USER, PRIV, WITH_GRANT_OPTION;
329336
USER HOST PRIV WITH_GRANT_OPTION
330337
mysql.session localhost BACKUP_ADMIN N
331338
mysql.session localhost CLONE_ADMIN N
339+
mysql.session localhost CONNECTION_ADMIN N
332340
mysql.session localhost PERSIST_RO_VARIABLES_ADMIN N
333341
mysql.session localhost SESSION_VARIABLES_ADMIN N
334342
mysql.session localhost SYSTEM_USER N
@@ -388,6 +396,7 @@ SELECT * FROM mysql.global_grants ORDER BY USER, PRIV, WITH_GRANT_OPTION;
388396
USER HOST PRIV WITH_GRANT_OPTION
389397
mysql.session localhost BACKUP_ADMIN N
390398
mysql.session localhost CLONE_ADMIN N
399+
mysql.session localhost CONNECTION_ADMIN N
391400
mysql.session localhost PERSIST_RO_VARIABLES_ADMIN N
392401
mysql.session localhost SESSION_VARIABLES_ADMIN N
393402
mysql.session localhost SYSTEM_USER N
@@ -884,6 +893,7 @@ GRANT SESSION_VARIABLES_ADMIN ON *.* TO 'mysql.session'@localhost;
884893
GRANT PERSIST_RO_VARIABLES_ADMIN ON *.* TO 'mysql.session'@localhost;
885894
GRANT CLONE_ADMIN ON *.* TO 'mysql.session'@localhost;
886895
GRANT BACKUP_ADMIN ON *.* TO 'mysql.session'@localhost;
896+
GRANT CONNECTION_ADMIN ON *.* TO 'mysql.session'@localhost;
887897
GRANT SYSTEM_USER ON *.* TO 'mysql.session'@localhost;
888898
# restart
889899
DROP USER u1@localhost;

‎mysql-test/r/grant_dynamic_session_variables_admin.result

+1-1
Original file line numberDiff line numberDiff line change
@@ -191,7 +191,7 @@ DROP USER wl12217@localhost;
191191
SHOW GRANTS for "mysql.session"@localhost;
192192
Grants for mysql.session@localhost
193193
GRANT SHUTDOWN, SUPER ON *.* TO `mysql.session`@`localhost`
194-
GRANT BACKUP_ADMIN,CLONE_ADMIN,PERSIST_RO_VARIABLES_ADMIN,SESSION_VARIABLES_ADMIN,SYSTEM_USER,SYSTEM_VARIABLES_ADMIN ON *.* TO `mysql.session`@`localhost`
194+
GRANT BACKUP_ADMIN,CLONE_ADMIN,CONNECTION_ADMIN,PERSIST_RO_VARIABLES_ADMIN,SESSION_VARIABLES_ADMIN,SYSTEM_USER,SYSTEM_VARIABLES_ADMIN ON *.* TO `mysql.session`@`localhost`
195195
GRANT SELECT ON `performance_schema`.* TO `mysql.session`@`localhost`
196196
GRANT SELECT ON `mysql`.`user` TO `mysql.session`@`localhost`
197197
#

‎mysql-test/r/mysql_upgrade.result

+2-2
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ Pattern "Could not open server upgrade info file \'.*\' for writing\. Please mak
1818
SHOW GRANTS FOR "mysql.session"@localhost;
1919
Grants for mysql.session@localhost
2020
GRANT SHUTDOWN, SUPER ON *.* TO `mysql.session`@`localhost`
21-
GRANT BACKUP_ADMIN,CLONE_ADMIN,PERSIST_RO_VARIABLES_ADMIN,SESSION_VARIABLES_ADMIN,SYSTEM_USER,SYSTEM_VARIABLES_ADMIN ON *.* TO `mysql.session`@`localhost`
21+
GRANT BACKUP_ADMIN,CLONE_ADMIN,CONNECTION_ADMIN,PERSIST_RO_VARIABLES_ADMIN,SESSION_VARIABLES_ADMIN,SYSTEM_USER,SYSTEM_VARIABLES_ADMIN ON *.* TO `mysql.session`@`localhost`
2222
GRANT SELECT ON `performance_schema`.* TO `mysql.session`@`localhost`
2323
GRANT SELECT ON `mysql`.`user` TO `mysql.session`@`localhost`
2424
REVOKE SESSION_VARIABLES_ADMIN ON *.* FROM "mysql.session"@localhost;
@@ -27,7 +27,7 @@ REVOKE SESSION_VARIABLES_ADMIN ON *.* FROM "mysql.session"@localhost;
2727
SHOW GRANTS FOR "mysql.session"@localhost;
2828
Grants for mysql.session@localhost
2929
GRANT SHUTDOWN, SUPER ON *.* TO `mysql.session`@`localhost`
30-
GRANT BACKUP_ADMIN,CLONE_ADMIN,PERSIST_RO_VARIABLES_ADMIN,SESSION_VARIABLES_ADMIN,SYSTEM_USER,SYSTEM_VARIABLES_ADMIN ON *.* TO `mysql.session`@`localhost`
30+
GRANT BACKUP_ADMIN,CLONE_ADMIN,CONNECTION_ADMIN,PERSIST_RO_VARIABLES_ADMIN,SESSION_VARIABLES_ADMIN,SYSTEM_USER,SYSTEM_VARIABLES_ADMIN ON *.* TO `mysql.session`@`localhost`
3131
GRANT SELECT ON `performance_schema`.* TO `mysql.session`@`localhost`
3232
GRANT SELECT ON `mysql`.`user` TO `mysql.session`@`localhost`
3333
#

‎mysql-test/suite/group_replication/r/gr_autorejoin_majority_loss_rejoin_failed_offline_mode.result

+95
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,95 @@
1+
include/group_replication.inc [rpl_server_count=3]
2+
Warnings:
3+
Note #### Sending passwords in plain text without SSL/TLS is extremely insecure.
4+
Note #### Storing MySQL user name or password information in the master info repository is not secure and is therefore not recommended. Please consider using the USER and PASSWORD connection options for START SLAVE; see the 'START SLAVE Syntax' in the MySQL Manual for more information.
5+
[connection server1]
6+
7+
#####
8+
# 0) The test requires three servers.
9+
#####
10+
11+
SET SESSION sql_log_bin = 0;
12+
call mtr.add_suppression("This member could not reach a majority of the members for more than 10 seconds. The member will now leave the group as instructed by the group_replication_unreachable_majority_timeout option.");
13+
call mtr.add_suppression("The server was automatically set into read only mode after an error was detected.");
14+
call mtr.add_suppression("The server was automatically set into offline mode after an error was detected.");
15+
call mtr.add_suppression("\\[GCS\\] Timeout while waiting for the group communication engine to exit!");
16+
call mtr.add_suppression("\\[GCS\\] The member has failed to gracefully leave the group.");
17+
call mtr.add_suppression("Started auto-rejoin procedure attempt*");
18+
call mtr.add_suppression("Auto-rejoin procedure attempt*");
19+
call mtr.add_suppression("\\[GCS\\] Error connecting to all peers. Member join failed. Local port:*");
20+
call mtr.add_suppression("\\[GCS\\] The member was unable to join the group.*");
21+
call mtr.add_suppression("Timeout while waiting for a view change event during the auto-rejoin procedure");
22+
call mtr.add_suppression("Unable to confirm whether the server has left the group or not. Check performance_schema.replication_group_members to check group membership information.");
23+
SET SESSION sql_log_bin = 1;
24+
include/gr_autorejoin_monitoring.inc
25+
SET @debug_saved = @@GLOBAL.DEBUG;
26+
SET @@GLOBAL.DEBUG='+d,group_replication_rejoin_short_retry';
27+
SET @@GLOBAL.DEBUG='+d,group_replication_stop_before_rejoin_loop';
28+
SET @@GLOBAL.DEBUG='+d,group_replication_stop_before_rejoin';
29+
SET @saved_exit_state_action = @@GLOBAL.group_replication_exit_state_action;
30+
SET @@GLOBAL.group_replication_exit_state_action = OFFLINE_MODE;
31+
include/start_and_bootstrap_group_replication.inc
32+
[connection server2]
33+
include/start_group_replication.inc
34+
[connection server3]
35+
include/start_group_replication.inc
36+
37+
####
38+
# 1) Provoke a majority loss.
39+
####
40+
41+
[connection server1]
42+
SET GLOBAL group_replication_autorejoin_tries = 3;
43+
include/gr_provoke_majority_loss.inc
44+
45+
####
46+
# 2) Verify that the member in the partitioned group will try to rejoin
47+
# the group the number of it is configured in
48+
# group_replication_autorejoin_tries sysvar.
49+
####
50+
51+
SET DEBUG_SYNC = "now WAIT_FOR signal.autorejoin_entering_loop";
52+
include/assert.inc [Auto-rejoin should be running]
53+
SET DEBUG_SYNC = "now SIGNAL signal.autorejoin_enter_loop";
54+
SET DEBUG_SYNC = "now WAIT_FOR signal.autorejoin_waiting";
55+
include/assert.inc [Auto-rejoin should be running]
56+
include/assert.inc [super_read_only should be enabled]
57+
include/assert.inc [offline_mode must be disabled]
58+
include/assert.inc [member state must be ERROR]
59+
include/assert.inc [We should have attempted 1 rejoins]
60+
SET DEBUG_SYNC = "now SIGNAL signal.autorejoin_continue";
61+
SET DEBUG_SYNC = "now WAIT_FOR signal.autorejoin_waiting";
62+
include/assert.inc [Auto-rejoin should be running]
63+
include/assert.inc [super_read_only should be enabled]
64+
include/assert.inc [offline_mode must be disabled]
65+
include/assert.inc [member state must be ERROR]
66+
include/assert.inc [We should have attempted 2 rejoins]
67+
SET DEBUG_SYNC = "now SIGNAL signal.autorejoin_continue";
68+
SET DEBUG_SYNC = "now WAIT_FOR signal.autorejoin_waiting";
69+
include/assert.inc [Auto-rejoin should be running]
70+
include/assert.inc [super_read_only should be enabled]
71+
include/assert.inc [offline_mode must be disabled]
72+
include/assert.inc [member state must be ERROR]
73+
include/assert.inc [We should have attempted 3 rejoins]
74+
SET DEBUG_SYNC = "now SIGNAL signal.autorejoin_continue";
75+
76+
####
77+
# 3 Since group_replication_exit_state_action is set to OFFLINE_MODE, the member
78+
# will enter the ERROR state and enable both super_read_only and offline_mode.
79+
####
80+
81+
include/gr_wait_for_member_state.inc
82+
include/assert_and_disable_read_only.inc
83+
include/assert_and_disable_offline_mode.inc
84+
85+
####
86+
# 4) Cleanup.
87+
####
88+
89+
include/stop_group_replication.inc
90+
include/start_and_bootstrap_group_replication.inc
91+
SET @@GLOBAL.DEBUG = @debug_saved;
92+
SET @@GLOBAL.group_replication_autorejoin_tries = default;
93+
SET @@GLOBAL.group_replication_exit_state_action = @saved_exit_state_action;
94+
include/gr_end_autorejoin_monitoring.inc
95+
include/group_replication_end.inc

‎mysql-test/suite/group_replication/r/gr_autorejoin_member_expel_rejoin_failed_offline_mode.result

+107
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,107 @@
1+
include/group_replication.inc [rpl_server_count=3]
2+
Warnings:
3+
Note #### Sending passwords in plain text without SSL/TLS is extremely insecure.
4+
Note #### Storing MySQL user name or password information in the master info repository is not secure and is therefore not recommended. Please consider using the USER and PASSWORD connection options for START SLAVE; see the 'START SLAVE Syntax' in the MySQL Manual for more information.
5+
[connection server1]
6+
7+
####
8+
# 0) The test requires three servers.
9+
####
10+
11+
SET sql_log_bin = 0;
12+
call mtr.add_suppression("Member was expelled from the group due to network failures, changing member status to ERROR.");
13+
call mtr.add_suppression("Started auto-rejoin procedure attempt*");
14+
call mtr.add_suppression("Auto-rejoin procedure attempt*");
15+
call mtr.add_suppression("The server was automatically set into offline mode after an error was detected.");
16+
call mtr.add_suppression("Unable to confirm whether the server has left the group or not. Check performance_schema.replication_group_members to check group membership information.");
17+
SET sql_log_bin = 1;
18+
include/gr_autorejoin_monitoring.inc
19+
SET @debug_saved = @@GLOBAL.DEBUG;
20+
SET @@GLOBAL.DEBUG='+d,group_replication_rejoin_short_retry';
21+
SET @@GLOBAL.DEBUG='+d,group_replication_fail_rejoin';
22+
SET @@GLOBAL.DEBUG='+d,group_replication_stop_before_rejoin_loop';
23+
SET @@GLOBAL.DEBUG='+d,group_replication_stop_before_rejoin';
24+
SET @saved_exit_state_action = @@GLOBAL.group_replication_exit_state_action;
25+
SET @@GLOBAL.group_replication_exit_state_action = OFFLINE_MODE;
26+
include/start_and_bootstrap_group_replication.inc
27+
[connection server2]
28+
include/start_group_replication.inc
29+
[connection server3]
30+
include/start_group_replication.inc
31+
32+
####
33+
# 1) Expel one of the members.
34+
####
35+
36+
[connection server1]
37+
SET GLOBAL group_replication_autorejoin_tries = 3;
38+
include/gr_expel_member_from_group.inc
39+
SET SESSION sql_log_bin = 0;
40+
CREATE TABLE pid_table(pid_no INT PRIMARY KEY);
41+
LOAD DATA LOCAL INFILE 'pid_file' INTO TABLE pid_table;
42+
DROP TABLE pid_table;
43+
SET SESSION sql_log_bin = 1;
44+
45+
####
46+
# 2) Verify that the expelled member will try to rejoin the group the
47+
# number of it is configured in group_replication_autorejoin_tries
48+
# sysvar.
49+
####
50+
51+
SET DEBUG_SYNC = "now WAIT_FOR signal.autorejoin_entering_loop";
52+
include/assert.inc [Auto-rejoin should be running]
53+
SET DEBUG_SYNC = "now SIGNAL signal.autorejoin_enter_loop";
54+
SET DEBUG_SYNC = "now WAIT_FOR signal.autorejoin_waiting";
55+
include/assert.inc [Auto-rejoin should be running]
56+
include/assert.inc [super_read_only should be enabled]
57+
include/assert.inc [offline_mode must be disabled]
58+
include/assert.inc [member state must be ERROR]
59+
include/assert.inc [We should have attempted 1 rejoins]
60+
SET DEBUG_SYNC = "now SIGNAL signal.autorejoin_continue";
61+
SET DEBUG_SYNC = "now WAIT_FOR signal.autorejoin_waiting";
62+
include/assert.inc [Auto-rejoin should be running]
63+
include/assert.inc [super_read_only should be enabled]
64+
include/assert.inc [offline_mode must be disabled]
65+
include/assert.inc [member state must be ERROR]
66+
include/assert.inc [We should have attempted 2 rejoins]
67+
SET DEBUG_SYNC = "now SIGNAL signal.autorejoin_continue";
68+
SET DEBUG_SYNC = "now WAIT_FOR signal.autorejoin_waiting";
69+
include/assert.inc [Auto-rejoin should be running]
70+
include/assert.inc [super_read_only should be enabled]
71+
include/assert.inc [offline_mode must be disabled]
72+
include/assert.inc [member state must be ERROR]
73+
include/assert.inc [We should have attempted 3 rejoins]
74+
SET DEBUG_SYNC = "now SIGNAL signal.autorejoin_continue";
75+
76+
####
77+
# 3) Verify that, with group_replication_exit_state_action set to OFFLINE_MODE,
78+
# the expelled member will enter the ERROR state and enable both
79+
# super_read_only and offline_mode mode.
80+
####
81+
82+
include/gr_wait_for_member_state.inc
83+
include/assert_and_disable_read_only.inc
84+
include/assert_and_disable_offline_mode.inc
85+
86+
####
87+
# 4) Manually rejoin the server to the group.
88+
####
89+
90+
include/stop_group_replication.inc
91+
include/start_group_replication.inc
92+
include/rpl_gr_wait_for_number_of_members.inc
93+
[connection server2]
94+
include/rpl_gr_wait_for_number_of_members.inc
95+
[connection server3]
96+
include/rpl_gr_wait_for_number_of_members.inc
97+
98+
####
99+
# 5) Cleanup.
100+
####
101+
102+
[connection server1]
103+
SET @@GLOBAL.DEBUG = @debug_saved;
104+
SET @@GLOBAL.group_replication_autorejoin_tries = default;
105+
SET @@GLOBAL.group_replication_exit_state_action = @saved_exit_state_action;
106+
include/gr_end_autorejoin_monitoring.inc
107+
include/group_replication_end.inc

0 commit comments

Comments
 (0)