Skip to content

Commit 852d7b3

Browse files
author
Ajo Robert
committed
Bug #20201006 spamming show processlist prevents old
connection threads from cleaning up. Analysis ========= Issue here is, delay in connection cleanup for which global connection counter is already decremented, makes room for new connections. Hence more than expected connections are observed in the server. Connections running statement "SHOW PROCESSLIST" or "SELECT on INFORMATION_SCHEMA.PROCESSLIST" acquires mutex LOCK_thd_remove for reading information of all the connections in server. Connections in cleanup phase, acquires mutex to remove thread from global thread list. Many such concurrent connections increases contention on mutex LOCK_thd_remove. In connection cleanup phase, connection count is decreased first and then the thd is removed from global thd list. This order makes new connection (above max_connections) possible while existing connections removal is still pending because of mutex LOCK_thd_remove being held by show processlist. Fix: ===== In connection clean phase, thd is removed from the global thd list first and then global connection count is decremented. Added code to wait for connection_count be be zero during shutdown to ensure connection threads are done with their task. Because of this change, tests which verifies PROCESSLIST and then status variable "threads_connected" might fail on slower machines. As part of this patch, changed test cases to wait until "threads_connected" is set to expected value instead of PROCESSLIST count.
2 parents cf3d43d + 3b633ba commit 852d7b3

20 files changed

+207
-60
lines changed

‎mysql-test/r/connect_debug.result

+49
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
2+
# -- Bug#20201006: Spamming show processlist prevents old connection
3+
# -- threads from cleaning up.
4+
SET @saved_max_connections = @@global.max_connections;
5+
SET GLOBAL max_connections = 2;
6+
7+
# -- Check that we allow only max_connections + 1 connections here
8+
connect con_1, localhost, root;
9+
connect con_2, localhost, root;
10+
connect(localhost,root,,test,MYSQL_PORT,MYSQL_SOCK);
11+
connect con_3, localhost, root;
12+
ERROR HY000: Too many connections
13+
14+
# -- Ensure we have max_connections + 1 connections.
15+
SELECT count(*)= @@global.max_connections + 1 FROM information_schema.processlist;
16+
count(*)= @@global.max_connections + 1
17+
1
18+
19+
# -- Take LOCK_thd_remove and close one connection then
20+
# attempt new one [should fail]...
21+
SET DEBUG_SYNC='inside_do_for_all_thd_copy SIGNAL disconnect_connection WAIT_FOR continue';
22+
SELECT user FROM INFORMATION_SCHEMA.PROCESSLIST GROUP BY user;;
23+
connection default;
24+
SET DEBUG_SYNC='now WAIT_FOR disconnect_connection';
25+
disconnect con_1;
26+
connect(localhost,root,,test,MYSQL_PORT,MYSQL_SOCK);
27+
connect con_3, localhost, root;
28+
ERROR HY000: Too many connections
29+
30+
# -- Release the lock. Now new connection should go through
31+
SET DEBUG_SYNC='now SIGNAL continue';
32+
connection con_2;
33+
user
34+
root
35+
SET DEBUG_SYNC='RESET';
36+
37+
# -- Waiting for connection to close...
38+
connect con_3, localhost, root;
39+
40+
# -- Closing connections...
41+
disconnect con_3;
42+
disconnect con_2;
43+
connection default;
44+
45+
# -- Resetting variables...
46+
SET GLOBAL max_connections= @saved_max_connections;
47+
48+
# -- End of Bug#20201006.
49+

‎mysql-test/r/server_offline_1.result

-5
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,4 @@
11
SET @global_saved_tmp = @@global.offline_mode;
2-
SELECT @user_count:=COUNT(USER) FROM INFORMATION_SCHEMA.PROCESSLIST;
3-
@user_count:=COUNT(USER)
4-
1
52
CREATE USER 'user1'@'localhost';
63
CREATE USER 'user2'@'localhost';
74
CREATE USER 'user3'@'localhost';
@@ -56,8 +53,6 @@ connect conu1,localhost,user1;
5653
connect conu2,localhost,user2;
5754
connect conu3,localhost,user3;
5855
connection default;
59-
SELECT @user_count;
60-
@user_count
6156
1
6257
SELECT COUNT(USER) FROM INFORMATION_SCHEMA.PROCESSLIST;
6358
COUNT(USER)

‎mysql-test/r/server_offline_2.result

-2
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
SELECT * FROM INFORMATION_SCHEMA.GLOBAL_VARIABLES WHERE variable_name LIKE '%offline_mode%';
22
VARIABLE_NAME VARIABLE_VALUE
33
OFFLINE_MODE OFF
4-
SET @user_count= 1;
54
CREATE USER 'user1'@'localhost';
65
CREATE USER 'user2'@'localhost';
76
CREATE USER 'user3'@'localhost';
@@ -27,7 +26,6 @@ disconnect conu1;
2726
ERROR HY000: The server is currently in offline mode
2827
connect(conu1,localhost,user1)
2928
connect(root,localhost,root)
30-
SET @user_count= 1;
3129
SELECT COUNT(USER) FROM INFORMATION_SCHEMA.PROCESSLIST;
3230
COUNT(USER)
3331
1

‎mysql-test/r/server_offline_3.result

-3
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,6 @@
11
SET @global_saved_tmp = @@global.offline_mode;
22
SET @global_autocommit = @@global.autocommit;
33
SET @@global.autocommit= OFF;
4-
SELECT @user_count:=COUNT(USER) FROM INFORMATION_SCHEMA.PROCESSLIST;
5-
@user_count:=COUNT(USER)
6-
1
74
CREATE USER 'user1'@'localhost';
85
CREATE USER 'user2'@'localhost';
96
connect conu1,localhost,user1;

‎mysql-test/r/server_offline_4.result

-3
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,4 @@
11
SET @global_saved_tmp = @@global.offline_mode;
2-
SELECT @user_count:=COUNT(USER) FROM INFORMATION_SCHEMA.PROCESSLIST;
3-
@user_count:=COUNT(USER)
4-
1
52
CREATE USER 'user1'@'localhost';
63
connect conu1,localhost,user1;
74
use test;

‎mysql-test/r/server_offline_5.result

-3
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,4 @@
11
SET @global_saved_tmp = @@global.offline_mode;
2-
SELECT @user_count:=COUNT(USER) FROM INFORMATION_SCHEMA.PROCESSLIST;
3-
@user_count:=COUNT(USER)
4-
1
52
CREATE USER 'user100'@'localhost';
63
CREATE USER 'user99'@'localhost';
74
CREATE USER 'user98'@'localhost';

‎mysql-test/r/server_offline_6.result

-3
Original file line numberDiff line numberDiff line change
@@ -2,9 +2,6 @@ SELECT * FROM INFORMATION_SCHEMA.GLOBAL_VARIABLES WHERE variable_name LIKE '%off
22
VARIABLE_NAME VARIABLE_VALUE
33
OFFLINE_MODE ON
44
SET @global_saved_tmp = @@global.offline_mode;
5-
SELECT @user_count:=COUNT(USER) FROM INFORMATION_SCHEMA.PROCESSLIST;
6-
@user_count:=COUNT(USER)
7-
1
85
CREATE USER 'user1'@'localhost';
96
SELECT COUNT(USER) FROM INFORMATION_SCHEMA.PROCESSLIST;
107
COUNT(USER)

‎mysql-test/suite/perfschema/r/dml_setup_instruments.result

+1-1
Original file line numberDiff line numberDiff line change
@@ -41,14 +41,14 @@ order by name limit 10;
4141
NAME ENABLED TIMED
4242
wait/synch/cond/sql/Commit_order_manager::m_workers.cond YES YES
4343
wait/synch/cond/sql/COND_compress_gtid_table YES YES
44+
wait/synch/cond/sql/COND_connection_count YES YES
4445
wait/synch/cond/sql/COND_flush_thread_cache YES YES
4546
wait/synch/cond/sql/COND_manager YES YES
4647
wait/synch/cond/sql/COND_queue_state YES YES
4748
wait/synch/cond/sql/COND_server_started YES YES
4849
wait/synch/cond/sql/COND_thd_list YES YES
4950
wait/synch/cond/sql/COND_thread_cache YES YES
5051
wait/synch/cond/sql/COND_thr_lock YES YES
51-
wait/synch/cond/sql/Event_scheduler::COND_state YES YES
5252
select * from performance_schema.setup_instruments
5353
where name='Wait';
5454
select * from performance_schema.setup_instruments

‎mysql-test/t/connect_debug.test

+79
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,79 @@
1+
2+
# This test makes no sense with the embedded server
3+
--source include/not_embedded.inc
4+
5+
--source include/have_debug_sync.inc
6+
7+
# Save the initial number of concurrent sessions
8+
--source include/count_sessions.inc
9+
10+
--echo
11+
--echo # -- Bug#20201006: Spamming show processlist prevents old connection
12+
--echo # -- threads from cleaning up.
13+
14+
--enable_connect_log
15+
SET @saved_max_connections = @@global.max_connections;
16+
SET GLOBAL max_connections = 2;
17+
18+
--echo
19+
--echo # -- Check that we allow only max_connections + 1 connections here
20+
--connect (con_1, localhost, root)
21+
--connect (con_2, localhost, root)
22+
--replace_result $MASTER_MYPORT MYSQL_PORT $MASTER_MYSOCK MYSQL_SOCK
23+
--error ER_CON_COUNT_ERROR
24+
--connect (con_3, localhost, root)
25+
26+
--echo
27+
--echo # -- Ensure we have max_connections + 1 connections.
28+
SELECT count(*)= @@global.max_connections + 1 FROM information_schema.processlist;
29+
30+
--echo
31+
--echo # -- Take LOCK_thd_remove and close one connection then
32+
--echo # attempt new one [should fail]...
33+
SET DEBUG_SYNC='inside_do_for_all_thd_copy SIGNAL disconnect_connection WAIT_FOR continue';
34+
--send SELECT user FROM INFORMATION_SCHEMA.PROCESSLIST GROUP BY user;
35+
36+
--connection default
37+
SET DEBUG_SYNC='now WAIT_FOR disconnect_connection';
38+
--disconnect con_1
39+
40+
--replace_result $MASTER_MYPORT MYSQL_PORT $MASTER_MYSOCK MYSQL_SOCK
41+
--error ER_CON_COUNT_ERROR
42+
--connect (con_3, localhost, root)
43+
44+
--echo
45+
--echo # -- Release the lock. Now new connection should go through
46+
SET DEBUG_SYNC='now SIGNAL continue';
47+
--connection con_2
48+
reap;
49+
50+
SET DEBUG_SYNC='RESET';
51+
52+
--echo
53+
--echo # -- Waiting for connection to close...
54+
let $count_sessions= 2;
55+
--source include/wait_until_count_sessions.inc
56+
57+
--connect (con_3, localhost, root)
58+
59+
--echo
60+
--echo # -- Closing connections...
61+
--disconnect con_3
62+
--disconnect con_2
63+
--source include/wait_until_disconnected.inc
64+
65+
--connection default
66+
67+
--echo
68+
--echo # -- Resetting variables...
69+
SET GLOBAL max_connections= @saved_max_connections;
70+
71+
--disable_connect_log
72+
73+
--echo
74+
--echo # -- End of Bug#20201006.
75+
--echo
76+
77+
# Wait till all disconnects are completed
78+
let $count_sessions= 1;
79+
--source include/wait_until_count_sessions.inc

‎mysql-test/t/server_offline_1.test

+8-8
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ SET @global_saved_tmp = @@global.offline_mode;
1010
--enable_connect_log
1111
# This count may not be 1, because of the probably existing connections
1212
# from the previous/parallel test runs
13-
SELECT @user_count:=COUNT(USER) FROM INFORMATION_SCHEMA.PROCESSLIST;
13+
let $user_count= `SELECT COUNT(USER) FROM INFORMATION_SCHEMA.PROCESSLIST`;
1414

1515
# test case 2.1.1 1)
1616
# Create 3 non-super users
@@ -33,8 +33,8 @@ SHOW STATUS LIKE 'threads_connected';
3333
SET GLOBAL offline_mode = ON;
3434

3535
# Wait until all non super user have been disconnected (for slow machines)
36-
let $wait_condition=SELECT COUNT(USER) = @user_count FROM INFORMATION_SCHEMA.PROCESSLIST;
37-
--source include/wait_condition.inc
36+
let $count_sessions= $user_count;
37+
--source include/wait_until_count_sessions.inc
3838

3939
SELECT COUNT(USER) FROM INFORMATION_SCHEMA.PROCESSLIST;
4040
SHOW STATUS LIKE 'threads_connected';
@@ -70,15 +70,15 @@ SHOW STATUS LIKE 'threads_connected';
7070
--connect(conu3,localhost,user3)
7171

7272
--connection default
73-
SELECT @user_count;
73+
echo $user_count;
7474
SELECT COUNT(USER) FROM INFORMATION_SCHEMA.PROCESSLIST;
7575
SHOW STATUS LIKE 'threads_connected';
7676

7777
SET GLOBAL offline_mode = ON;
7878

7979
# Wait until all non super user have been disconnected (for slow machines)
80-
let $wait_condition=SELECT COUNT(USER) = 1+@user_count FROM INFORMATION_SCHEMA.PROCESSLIST;
81-
--source include/wait_condition.inc
80+
let $count_sessions= 1 + $user_count;
81+
--source include/wait_until_count_sessions.inc
8282

8383
SELECT COUNT(USER) FROM INFORMATION_SCHEMA.PROCESSLIST;
8484
SHOW STATUS LIKE 'threads_connected';
@@ -90,8 +90,8 @@ SHOW STATUS LIKE 'threads_connected';
9090
--disconnect conu4
9191

9292
# Wait until all users have been disconnected (for slow machines)
93-
let $wait_condition=SELECT COUNT(USER) = @user_count FROM INFORMATION_SCHEMA.PROCESSLIST;
94-
--source include/wait_condition.inc
93+
let $count_sessions= $user_count;
94+
--source include/wait_until_count_sessions.inc
9595

9696
DROP USER 'user1'@'localhost';
9797
DROP USER 'user2'@'localhost';

‎mysql-test/t/server_offline_2.test

+4-6
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,6 @@ let $global_saved_tmp = `SELECT @@global.offline_mode`;
1515
# This count may not be 1, because of the probably existing connections
1616
# from the previous/parallel test runs
1717
let $user_count= `SELECT COUNT(USER) FROM INFORMATION_SCHEMA.PROCESSLIST`;
18-
--eval SET @user_count= $user_count
1918

2019
# Create 3 non-super users
2120

@@ -34,8 +33,8 @@ SHOW STATUS LIKE 'threads_connected';
3433
SET GLOBAL offline_mode = ON;
3534

3635
# Wait until all non super user have been disconnected (for slow machines)
37-
let $wait_condition=SELECT COUNT(USER) = @user_count FROM INFORMATION_SCHEMA.PROCESSLIST;
38-
--source include/wait_condition.inc
36+
let $count_sessions= $user_count;
37+
--source include/wait_until_count_sessions.inc
3938

4039
SELECT COUNT(USER) FROM INFORMATION_SCHEMA.PROCESSLIST;
4140
SHOW STATUS LIKE 'threads_connected';
@@ -54,9 +53,8 @@ SHOW STATUS LIKE 'threads_connected';
5453
--connect(root,localhost,root)
5554
--echo connect(root,localhost,root)
5655
# Wait until all non super user have been disconnected (for slow machines)
57-
--eval SET @user_count= $user_count
58-
let $wait_condition=SELECT COUNT(USER) = @user_count FROM INFORMATION_SCHEMA.PROCESSLIST;
59-
--source include/wait_condition.inc
56+
let $count_sessions= $user_count;
57+
--source include/wait_until_count_sessions.inc
6058
SELECT COUNT(USER) FROM INFORMATION_SCHEMA.PROCESSLIST;
6159

6260
--echo connect(default,localhost,root)

‎mysql-test/t/server_offline_3.test

+7-7
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ SET @@global.autocommit= OFF;
1212
--enable_connect_log
1313
# This count may not be 1, because of the probably existing connections
1414
# from the previous/parallel test runs
15-
SELECT @user_count:=COUNT(USER) FROM INFORMATION_SCHEMA.PROCESSLIST;
15+
let $user_count= `SELECT COUNT(USER) FROM INFORMATION_SCHEMA.PROCESSLIST`;
1616

1717
# test case 2.1.1 3)
1818
# Create 3 non-super users
@@ -42,8 +42,8 @@ INSERT INTO t1 VALUES (2,'bbbbbbbbbb');
4242
--connection default
4343
SET GLOBAL offline_mode = ON;
4444
# Wait until all non super user have been disconnected (for slow machines)
45-
let $wait_condition=SELECT COUNT(USER) = @user_count FROM INFORMATION_SCHEMA.PROCESSLIST;
46-
--source include/wait_condition.inc
45+
let $count_sessions= $user_count;
46+
--source include/wait_until_count_sessions.inc
4747
SELECT COUNT(USER) FROM INFORMATION_SCHEMA.PROCESSLIST;
4848
SET GLOBAL offline_mode = OFF;
4949

@@ -61,16 +61,16 @@ DROP TABLE t1;
6161
--connection default
6262
SET GLOBAL offline_mode = ON;
6363
# Wait until all non super user have been disconnected (for slow machines)
64-
let $wait_condition=SELECT COUNT(USER) = @user_count FROM INFORMATION_SCHEMA.PROCESSLIST;
65-
--source include/wait_condition.inc
64+
let $count_sessions= $user_count;
65+
--source include/wait_until_count_sessions.inc
6666
SELECT COUNT(USER) FROM INFORMATION_SCHEMA.PROCESSLIST;
6767

6868
# Clean up
6969
--disconnect conu1
7070
--disconnect conu2
7171
# Wait until all users have been disconnected (for slow machines)
72-
let $wait_condition=SELECT COUNT(USER) = @user_count FROM INFORMATION_SCHEMA.PROCESSLIST;
73-
--source include/wait_condition.inc
72+
let $count_sessions= $user_count;
73+
--source include/wait_until_count_sessions.inc
7474

7575
DROP USER 'user1'@'localhost';
7676
DROP USER 'user2'@'localhost';

‎mysql-test/t/server_offline_4.test

+7-7
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ let $MYSQLD_DATADIR= `SELECT @@global.datadir`;
1111
--enable_connect_log
1212
# This count may not be 1, because of the probably existing connections
1313
# from the previous/parallel test runs
14-
SELECT @user_count:=COUNT(USER) FROM INFORMATION_SCHEMA.PROCESSLIST;
14+
let $user_count= `SELECT COUNT(USER) FROM INFORMATION_SCHEMA.PROCESSLIST`;
1515

1616
# test case 2.1.1 3)
1717
CREATE USER 'user1'@'localhost';
@@ -42,8 +42,8 @@ let $wait_condition=
4242
# Alter will completely be rolled back (No temp files available)
4343
SET GLOBAL offline_mode = ON;
4444
# Wait until all non super user have been disconnected (for slow machines)
45-
let $wait_condition=SELECT COUNT(USER) = @user_count FROM INFORMATION_SCHEMA.PROCESSLIST;
46-
--source include/wait_condition.inc
45+
let $count_sessions= $user_count;
46+
--source include/wait_until_count_sessions.inc
4747
--list_files $MYSQLD_DATADIR/test t*
4848
SELECT COUNT(USER) FROM INFORMATION_SCHEMA.PROCESSLIST;
4949

@@ -59,15 +59,15 @@ DROP TABLE t2;
5959
--connection default
6060
SET GLOBAL offline_mode = ON;
6161
# Wait until all non super user have been disconnected (for slow machines)
62-
let $wait_condition=SELECT COUNT(USER) = @user_count FROM INFORMATION_SCHEMA.PROCESSLIST;
63-
--source include/wait_condition.inc
62+
let $count_sessions= $user_count;
63+
--source include/wait_until_count_sessions.inc
6464
SELECT COUNT(USER) FROM INFORMATION_SCHEMA.PROCESSLIST;
6565

6666
# Clean up
6767
--disconnect conu1
6868
# Wait until all users have been disconnected (for slow machines)
69-
let $wait_condition=SELECT COUNT(USER) = @user_count FROM INFORMATION_SCHEMA.PROCESSLIST;
70-
--source include/wait_condition.inc
69+
let $count_sessions= $user_count;
70+
--source include/wait_until_count_sessions.inc
7171

7272
DROP USER 'user1'@'localhost';
7373

0 commit comments

Comments
 (0)