Skip to content

Commit 4a3e309

Browse files
author
Ajo Robert
committed
Bug #20201006 spamming show processlist prevents old
connection threads from cleaning up. Analysis -------- Issue here is, delay in connection cleanup for which global connection counter is already decremented, makes room for new connections. Hence more than expected connections are observed in the server. Connections running statement "SHOW PROCESSLIST" or "SELECT on INFORMATION_SCHEMA.PROCESSLIST" acquires mutex LOCK_thd_remove for reading information of all the connections in server. Connections in cleanup phase, acquires mutex to remove thread from global thread list. Many such concurrent connections increases contention on mutex LOCK_thd_remove. In connection cleanup phase, connection count is decreased first and then the thd is removed from global thd list. This order makes new connection (above max_connections) possible while existing connections removal is still pending because of mutex LOCK_thd_remove. Fix: --- In connection clean phase, now first thd is removed from the global thd list and then the global connection count is decremented.
1 parent 739ac25 commit 4a3e309

File tree

5 files changed

+134
-3
lines changed

5 files changed

+134
-3
lines changed

‎mysql-test/r/connect_debug.result

+49
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
2+
# -- Bug#20201006: Spamming show processlist prevents old connection
3+
# -- threads from cleaning up.
4+
SET @saved_max_connections = @@global.max_connections;
5+
SET GLOBAL max_connections = 2;
6+
7+
# -- Check that we allow only max_connections + 1 connections here
8+
connect con_1, localhost, root;
9+
connect con_2, localhost, root;
10+
connect(localhost,root,,test,MYSQL_PORT,MYSQL_SOCK);
11+
connect con_3, localhost, root;
12+
ERROR HY000: Too many connections
13+
14+
# -- Ensure we have max_connections + 1 connections.
15+
SELECT count(*)= @@global.max_connections + 1 FROM information_schema.processlist;
16+
count(*)= @@global.max_connections + 1
17+
1
18+
19+
# -- Take LOCK_thd_remove and close one connection then
20+
# attempt new one [should fail]...
21+
SET DEBUG_SYNC='fill_schema_processlist_after_copying_threads SIGNAL disconnect_connection WAIT_FOR continue';
22+
SELECT user FROM INFORMATION_SCHEMA.PROCESSLIST GROUP BY user;;
23+
connection default;
24+
SET DEBUG_SYNC='now WAIT_FOR disconnect_connection';
25+
disconnect con_1;
26+
connect(localhost,root,,test,MYSQL_PORT,MYSQL_SOCK);
27+
connect con_3, localhost, root;
28+
ERROR HY000: Too many connections
29+
30+
# -- Release the lock. Now new connection should go through
31+
SET DEBUG_SYNC='now SIGNAL continue';
32+
connection con_2;
33+
user
34+
root
35+
SET DEBUG_SYNC='RESET';
36+
37+
# -- Waiting for connection to close...
38+
connect con_3, localhost, root;
39+
40+
# -- Closing connections...
41+
disconnect con_3;
42+
disconnect con_2;
43+
connection default;
44+
45+
# -- Resetting variables...
46+
SET GLOBAL max_connections= @saved_max_connections;
47+
48+
# -- End of Bug#20201006.
49+

‎mysql-test/t/connect_debug.test

+79
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,79 @@
1+
2+
# This test makes no sense with the embedded server
3+
--source include/not_embedded.inc
4+
5+
--source include/have_debug_sync.inc
6+
7+
# Save the initial number of concurrent sessions
8+
--source include/count_sessions.inc
9+
10+
--echo
11+
--echo # -- Bug#20201006: Spamming show processlist prevents old connection
12+
--echo # -- threads from cleaning up.
13+
14+
--enable_connect_log
15+
SET @saved_max_connections = @@global.max_connections;
16+
SET GLOBAL max_connections = 2;
17+
18+
--echo
19+
--echo # -- Check that we allow only max_connections + 1 connections here
20+
--connect (con_1, localhost, root)
21+
--connect (con_2, localhost, root)
22+
--replace_result $MASTER_MYPORT MYSQL_PORT $MASTER_MYSOCK MYSQL_SOCK
23+
--error ER_CON_COUNT_ERROR
24+
--connect (con_3, localhost, root)
25+
26+
--echo
27+
--echo # -- Ensure we have max_connections + 1 connections.
28+
SELECT count(*)= @@global.max_connections + 1 FROM information_schema.processlist;
29+
30+
--echo
31+
--echo # -- Take LOCK_thd_remove and close one connection then
32+
--echo # attempt new one [should fail]...
33+
SET DEBUG_SYNC='fill_schema_processlist_after_copying_threads SIGNAL disconnect_connection WAIT_FOR continue';
34+
--send SELECT user FROM INFORMATION_SCHEMA.PROCESSLIST GROUP BY user;
35+
36+
--connection default
37+
SET DEBUG_SYNC='now WAIT_FOR disconnect_connection';
38+
--disconnect con_1
39+
40+
--replace_result $MASTER_MYPORT MYSQL_PORT $MASTER_MYSOCK MYSQL_SOCK
41+
--error ER_CON_COUNT_ERROR
42+
--connect (con_3, localhost, root)
43+
44+
--echo
45+
--echo # -- Release the lock. Now new connection should go through
46+
SET DEBUG_SYNC='now SIGNAL continue';
47+
--connection con_2
48+
reap;
49+
50+
SET DEBUG_SYNC='RESET';
51+
52+
--echo
53+
--echo # -- Waiting for connection to close...
54+
let $wait_condition =
55+
SELECT COUNT(*) = 2
56+
FROM information_schema.processlist;
57+
--source include/wait_condition.inc
58+
59+
--connect (con_3, localhost, root)
60+
61+
--echo
62+
--echo # -- Closing connections...
63+
--disconnect con_3
64+
--disconnect con_2
65+
--source include/wait_until_disconnected.inc
66+
67+
--connection default
68+
--disable_connect_log
69+
70+
--echo
71+
--echo # -- Resetting variables...
72+
SET GLOBAL max_connections= @saved_max_connections;
73+
74+
--echo
75+
--echo # -- End of Bug#20201006.
76+
--echo
77+
78+
# Wait till all disconnects are completed
79+
--source include/wait_until_count_sessions.inc

‎sql/mysqld.cc

+1-1
Original file line numberDiff line numberDiff line change
@@ -2749,8 +2749,8 @@ bool one_thread_per_connection_end(THD *thd, bool block_pthread)
27492749
DBUG_PRINT("info", ("thd %p block_pthread %d", thd, (int) block_pthread));
27502750

27512751
thd->release_resources();
2752-
dec_connection_count();
27532752
remove_global_thread(thd);
2753+
dec_connection_count();
27542754
if (kill_blocked_pthreads_flag)
27552755
{
27562756
// Do not block if we are about to shut down

‎sql/scheduler.cc

+3-2
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
1-
/* Copyright (c) 2007, 2014, Oracle and/or its affiliates. All rights reserved.
1+
/* Copyright (c) 2007, 2015, Oracle and/or its affiliates. All rights
2+
reserved.
23
34
This program is free software; you can redistribute it and/or modify
45
it under the terms of the GNU General Public License as published by
@@ -33,8 +34,8 @@
3334
static bool no_threads_end(THD *thd, bool put_in_cache)
3435
{
3536
thd_release_resources(thd);
36-
dec_connection_count();
3737
remove_global_thread(thd);
38+
dec_connection_count();
3839
// THD is an incomplete type here, so use destroy_thd() to delete it.
3940
destroy_thd(thd);
4041

‎sql/sql_show.cc

+2
Original file line numberDiff line numberDiff line change
@@ -2222,6 +2222,8 @@ int fill_schema_processlist(THD* thd, TABLE_LIST* tables, Item* cond)
22222222
mysql_mutex_lock(&LOCK_thd_remove);
22232223
copy_global_thread_list(&global_thread_list_copy);
22242224

2225+
DEBUG_SYNC(thd,"fill_schema_processlist_after_copying_threads");
2226+
22252227
Thread_iterator it= global_thread_list_copy.begin();
22262228
Thread_iterator end= global_thread_list_copy.end();
22272229
for (; it != end; ++it)

0 commit comments

Comments
 (0)