Skip to content

Commit b4bf2d5

Browse files
thayumanavar77thayumanavar77
committed
WL#6606 - Offload THD and network initialization to worker
 thread. Initialization of THD and vio/net initialization happens in the acceptor thread that accepts the connection. THD and network initialization involves acquiring locks, memory allocation of various structures and system calls which are compute-bound as well as tasks that may block. The acceptor thread is an event loop that waits for new connection events from clients. To maximize the number of connections that can be handled per unit of time, the acceptor thread should spend as much of its time listening for new connections. This means thd and vio/net initialization should be offloaded from the accept event loop and delegated to worker threads that handle the client connections. This worklog provides a generic framework which offloads THD initialization and net/vio initialization to worker threads for all types of communication channels (shared memory, named pipes and sockets) that clients connect with server. In addition, this worklog refactored the existing interfaces of the struct scheduler_functions into an object oriented API, refactored and moved code related to connection handling and its management into a separate directory and files that contain implementations of specific related functionality. This resulted in removal of unnecessary #defines, modularity, better code clarity and readability in addition to performance improvements made in the worklog. As result of changes in this worklog, the follow bugs have been fixed: Bug#12951536 - THD INITIALIZATION TOO EXPENSIVE FOR ACCEPT() THREAD. Bug#12951595 - TOO MUCH NETWORK INITIALIZATION DONE IN ACCEPT() THREAD. Bug#12951605 - ACCEPT() SOCKET GETS TOO MUCH OF FCNTL()S. User Visible Changes: The system variables "bind_address", "thread_handling", ""thread_cache_size" and status variables "threads_cached", "Slow_launch_threads" are no longer visible in embedded server mode (where they have no effect).
1 parent b9c431e commit b4bf2d5

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

46 files changed

+3998
-2297
lines changed

‎include/mysql/plugin_audit.h.pp

+7-5
Original file line numberDiff line numberDiff line change
@@ -53,13 +53,15 @@
5353
void thd_wait_begin(void* thd, int wait_type);
5454
void thd_wait_end(void* thd);
5555
#include <mysql/service_thread_scheduler.h>
56-
struct scheduler_functions;
56+
struct Connection_handler_callback;
5757
extern struct my_thread_scheduler_service {
58-
int (*set)(struct scheduler_functions *scheduler);
59-
int (*reset)();
58+
int (*connection_handler_set)(void* conn_handler,
59+
struct Connection_handler_callback *cb);
60+
int (*connection_handler_reset)();
6061
} *my_thread_scheduler_service;
61-
int my_thread_scheduler_set(struct scheduler_functions *scheduler);
62-
int my_thread_scheduler_reset();
62+
int my_connection_handler_set(void* conn_handler,
63+
struct Connection_handler_callback *cb);
64+
int my_connection_handler_reset();
6365
#include <mysql/service_my_plugin_log.h>
6466
enum plugin_log_level
6567
{

‎include/mysql/plugin_auth.h.pp

+7-5
Original file line numberDiff line numberDiff line change
@@ -53,13 +53,15 @@
5353
void thd_wait_begin(void* thd, int wait_type);
5454
void thd_wait_end(void* thd);
5555
#include <mysql/service_thread_scheduler.h>
56-
struct scheduler_functions;
56+
struct Connection_handler_callback;
5757
extern struct my_thread_scheduler_service {
58-
int (*set)(struct scheduler_functions *scheduler);
59-
int (*reset)();
58+
int (*connection_handler_set)(void* conn_handler,
59+
struct Connection_handler_callback *cb);
60+
int (*connection_handler_reset)();
6061
} *my_thread_scheduler_service;
61-
int my_thread_scheduler_set(struct scheduler_functions *scheduler);
62-
int my_thread_scheduler_reset();
62+
int my_connection_handler_set(void* conn_handler,
63+
struct Connection_handler_callback *cb);
64+
int my_connection_handler_reset();
6365
#include <mysql/service_my_plugin_log.h>
6466
enum plugin_log_level
6567
{

‎include/mysql/plugin_ftparser.h.pp

+7-5
Original file line numberDiff line numberDiff line change
@@ -53,13 +53,15 @@
5353
void thd_wait_begin(void* thd, int wait_type);
5454
void thd_wait_end(void* thd);
5555
#include <mysql/service_thread_scheduler.h>
56-
struct scheduler_functions;
56+
struct Connection_handler_callback;
5757
extern struct my_thread_scheduler_service {
58-
int (*set)(struct scheduler_functions *scheduler);
59-
int (*reset)();
58+
int (*connection_handler_set)(void* conn_handler,
59+
struct Connection_handler_callback *cb);
60+
int (*connection_handler_reset)();
6061
} *my_thread_scheduler_service;
61-
int my_thread_scheduler_set(struct scheduler_functions *scheduler);
62-
int my_thread_scheduler_reset();
62+
int my_connection_handler_set(void* conn_handler,
63+
struct Connection_handler_callback *cb);
64+
int my_connection_handler_reset();
6365
#include <mysql/service_my_plugin_log.h>
6466
enum plugin_log_level
6567
{

‎include/mysql/psi/mysql_socket.h

+85
Original file line numberDiff line numberDiff line change
@@ -480,6 +480,19 @@ inline_mysql_socket_set_state(MYSQL_SOCKET socket, enum PSI_socket_state state)
480480
inline_mysql_socket_setsockopt(FD, LV, ON, OP, OL)
481481
#endif
482482

483+
/**
484+
@def mysql_sock_set_nonblocking
485+
Set socket to non-blocking.
486+
@param FD instrumented socket descriptor
487+
*/
488+
#ifdef HAVE_PSI_SOCKET_INTERFACE
489+
#define mysql_sock_set_nonblocking(FD) \
490+
inline_mysql_sock_set_nonblocking(__FILE__, __LINE__, FD)
491+
#else
492+
#define mysql_sock_set_nonblocking(FD) \
493+
inline_mysql_sock_set_nonblocking(FD)
494+
#endif
495+
483496
/**
484497
@def mysql_socket_listen(FD, N)
485498
Set socket state to listen for an incoming connection.
@@ -974,6 +987,78 @@ inline_mysql_socket_setsockopt
974987
return result;
975988
}
976989

990+
/** set_socket_nonblock */
991+
static inline int
992+
set_socket_nonblock(my_socket fd)
993+
{
994+
int ret= 0;
995+
#ifdef _WIN32
996+
{
997+
u_long nonblocking= 1;
998+
ret= ioctlsocket(fd, FIONBIO, &nonblocking);
999+
}
1000+
#else
1001+
{
1002+
int fd_flags;
1003+
fd_flags= fcntl(fd, F_GETFL, 0);
1004+
if (fd_flags < 0)
1005+
return errno;
1006+
#if defined(O_NONBLOCK)
1007+
fd_flags |= O_NONBLOCK;
1008+
#elif defined(O_NDELAY)
1009+
fd_flags |= O_NDELAY;
1010+
#elif defined(O_FNDELAY)
1011+
fd_flags |= O_FNDELAY;
1012+
#else
1013+
#error "No definition of non-blocking flag found."
1014+
#endif /* O_NONBLOCK */
1015+
if (fcntl(fd, F_SETFL, fd_flags) == -1)
1016+
ret= errno;
1017+
}
1018+
#endif /* _WIN32 */
1019+
return ret;
1020+
}
1021+
1022+
/** mysql_socket_set_nonblocking */
1023+
1024+
static inline int
1025+
inline_mysql_sock_set_nonblocking
1026+
(
1027+
#ifdef HAVE_PSI_SOCKET_INTERFACE
1028+
const char *src_file, uint src_line,
1029+
#endif
1030+
MYSQL_SOCKET mysql_socket
1031+
)
1032+
{
1033+
int result= 0;
1034+
1035+
#ifdef HAVE_PSI_SOCKET_INTERFACE
1036+
if (mysql_socket.m_psi)
1037+
{
1038+
/* Instrumentation start */
1039+
PSI_socket_locker *locker;
1040+
PSI_socket_locker_state state;
1041+
locker= PSI_SOCKET_CALL(start_socket_wait)
1042+
(&state, mysql_socket.m_psi, PSI_SOCKET_OPT,
1043+
(size_t)0, src_file, src_line);
1044+
1045+
/* Instrumented code */
1046+
result= set_socket_nonblock(mysql_socket.fd);
1047+
1048+
/* Instrumentation end */
1049+
if (locker != NULL)
1050+
PSI_SOCKET_CALL(end_socket_wait)(locker, (size_t)0);
1051+
1052+
return result;
1053+
}
1054+
#endif
1055+
1056+
/* Non instrumented code */
1057+
result= set_socket_nonblock(mysql_socket.fd);
1058+
1059+
return result;
1060+
}
1061+
9771062
/** mysql_socket_listen */
9781063

9791064
static inline int

‎include/mysql/service_thread_scheduler.h

+17-26
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
/*
2-
Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
2+
Copyright (c) 2010, 2013, Oracle and/or its affiliates. All rights reserved.
33
44
This program is free software; you can redistribute it and/or modify
55
it under the terms of the GNU General Public License as published by
@@ -18,43 +18,34 @@
1818
#ifndef SERVICE_THREAD_SCHEDULER_INCLUDED
1919
#define SERVICE_THREAD_SCHEDULER_INCLUDED
2020

21+
#ifdef __cplusplus
22+
class Connection_handler;
23+
#define MYSQL_CONNECTION_HANDLER Connection_handler*
24+
#else
25+
#define MYSQL_CONNECTION_HANDLER void*
26+
#endif
27+
2128
#ifdef __cplusplus
2229
extern "C" {
2330
#endif
2431

25-
struct scheduler_functions;
32+
struct Connection_handler_callback;
2633

2734
extern struct my_thread_scheduler_service {
28-
int (*set)(struct scheduler_functions *scheduler);
29-
int (*reset)();
35+
int (*connection_handler_set)(MYSQL_CONNECTION_HANDLER conn_handler,
36+
struct Connection_handler_callback *cb);
37+
int (*connection_handler_reset)();
3038
} *my_thread_scheduler_service;
31-
3239
#ifdef MYSQL_DYNAMIC_PLUGIN
3340

34-
#define my_thread_scheduler_set(F) my_thread_scheduler_service->set((F))
35-
#define my_thread_scheduler_reset() my_thread_scheduler_service->reset()
41+
#define my_connection_handler_set(F, M) my_thread_scheduler_service->connection_handler_set((F), (M))
42+
#define my_connection_handler_reset() my_thread_scheduler_service->connection_handler_reset()
3643

3744
#else
3845

39-
/**
40-
Set the thread scheduler to use for the server.
41-
42-
@param scheduler Pointer to scheduler callbacks to use.
43-
@retval 0 Scheduler installed correctly.
44-
@retval 1 Invalid value (NULL) used for scheduler.
45-
*/
46-
int my_thread_scheduler_set(struct scheduler_functions *scheduler);
47-
48-
/**
49-
Restore the previous thread scheduler.
50-
51-
@note If no thread scheduler was installed previously with
52-
thd_set_thread_scheduler, this function will report an error.
53-
54-
@retval 0 Scheduler installed correctly.
55-
@retval 1 No scheduler installed.
56-
*/
57-
int my_thread_scheduler_reset();
46+
int my_connection_handler_set(MYSQL_CONNECTION_HANDLER conn_handler,
47+
struct Connection_handler_callback *cb);
48+
int my_connection_handler_reset();
5849

5950
#endif
6051

‎include/mysql/thread_pool_priv.h

+10-4
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
/*
2-
Copyright (c) 2010, 2012, Oracle and/or its affiliates. All rights reserved.
2+
Copyright (c) 2010, 2013, Oracle and/or its affiliates. All rights reserved.
33
44
This program is free software; you can redistribute it and/or modify
55
it under the terms of the GNU General Public License as published by
@@ -31,13 +31,20 @@
3131
*/
3232
#include <mysqld_error.h> /* To get ER_ERROR_ON_READ */
3333
#define MYSQL_SERVER 1
34-
#include <scheduler.h>
34+
#include <conn_handler/channel_info.h>
35+
#include <conn_handler/connection_handler_manager.h>
3536
#include <debug_sync.h>
3637
#include <sql_profile.h>
3738
#include <table.h>
3839
#include <set>
3940

4041
typedef std::set<THD*>::iterator Thread_iterator;
42+
43+
/* create thd from channel_info object */
44+
THD* create_thd(Channel_info* channel_info);
45+
/* destroy channel_info object */
46+
void destroy_channel_info(Channel_info* channel_info);
47+
4148
/* Needed to get access to scheduler variables */
4249
void* thd_get_scheduler_data(THD *thd);
4350
void thd_set_scheduler_data(THD *thd, void *data);
@@ -91,8 +98,6 @@ bool do_command(THD *thd);
9198
*/
9299
/* Initialise a new connection handler thread */
93100
bool init_new_connection_handler_thread();
94-
/* Set up connection thread before use as execution thread */
95-
bool setup_connection_thread_globals(THD *thd);
96101
/* Prepare connection as part of connection set-up */
97102
bool thd_prepare_connection(THD *thd);
98103
/* Release auditing before executing statement */
@@ -129,6 +134,7 @@ void remove_global_thread(THD *thd);
129134
attributes.
130135
*/
131136
void inc_thread_created(void);
137+
void inc_aborted_connects(void);
132138
ulong get_max_connections(void);
133139
pthread_attr_t *get_connection_attrib(void);
134140
#endif

‎libmysqld/CMakeLists.txt

+1
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ INCLUDE_DIRECTORIES(
2222
${CMAKE_SOURCE_DIR}/libmysqld
2323
${CMAKE_SOURCE_DIR}/sql
2424
${CMAKE_SOURCE_DIR}/sql/auth
25+
${CMAKE_SOURCE_DIR}/sql/conn_handler
2526
${CMAKE_BINARY_DIR}/sql
2627
${CMAKE_SOURCE_DIR}/regex
2728
${ZLIB_INCLUDE_DIR}

‎mysql-test/r/per_thread_connection_handler.result

+43
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
SET @old_debug= @@GLOBAL.debug;
2+
SET @orig_max_connections= @@global.max_connections;
3+
connect con1, localhost, root,,mysql;
4+
SELECT 1;
5+
1
6+
1
7+
connect con2, localhost, root,,mysql;
8+
SELECT 1;
9+
1
10+
1
11+
connect con3, localhost, root,,mysql;
12+
SELECT 1;
13+
1
14+
1
15+
SET GLOBAL debug= '+d,fail_thread_create';
16+
connect(localhost,root,,mysql,MYSQL_PORT,MYSQL_SOCK);
17+
connect con4, localhost, root,,mysql;
18+
ERROR HY000: Can't create a new thread (errno 1); if you are not out of available memory, you can consult the manual for a possible OS-dependent bug
19+
connection default;
20+
SET GLOBAL debug="-d,fail_thread_create";
21+
disconnect con1;
22+
disconnect con2;
23+
disconnect con3;
24+
SET GLOBAL max_connections= 3;
25+
connect con1, localhost, root,,mysql;
26+
connect con2, localhost, root,,mysql;
27+
connect con3, localhost, root,,mysql;
28+
connect(localhost,root,,mysql,MYSQL_PORT,MYSQL_SOCK);
29+
connect con4, localhost, root,,mysql;
30+
ERROR HY000: Too many connections
31+
connection default;
32+
disconnect con1;
33+
disconnect con2;
34+
disconnect con3;
35+
SET GLOBAL debug= '+d,simulate_resource_failure';
36+
call mtr.add_suppression("Out of memory");
37+
connect(localhost,root,,mysql,MYSQL_PORT,MYSQL_SOCK);
38+
connect con1, localhost, root,,mysql;
39+
ERROR HY000: Lost connection to MySQL server at 'reading initial communication packet', system error: NUM
40+
connection default;
41+
SET GLOBAL debug= '-d,simulate_resource_failure';
42+
SET GLOBAL debug= @old_debug;
43+
SET GLOBAL max_connections= @orig_max_connections;

‎mysql-test/r/variables.result

-3
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,6 @@ set @my_query_cache_size =@@global.query_cache_size;
3333
set @my_server_id =@@global.server_id;
3434
set @my_slow_launch_time =@@global.slow_launch_time;
3535
set @my_storage_engine =@@global.default_storage_engine;
36-
set @my_thread_cache_size =@@global.thread_cache_size;
3736
set @my_max_allowed_packet =@@global.max_allowed_packet;
3837
set @my_join_buffer_size =@@global.join_buffer_size;
3938
set @`test`=1;
@@ -593,7 +592,6 @@ set sql_select_limit=default;
593592
set sql_warnings=1;
594593
set global table_open_cache=100;
595594
set default_storage_engine=myisam;
596-
set global thread_cache_size=100;
597595
set timestamp=1, timestamp=default;
598596
set tmp_table_size=100;
599597
Warnings:
@@ -1090,7 +1088,6 @@ set global query_cache_size =@my_query_cache_size;
10901088
set global server_id =@my_server_id;
10911089
set global slow_launch_time =@my_slow_launch_time;
10921090
set global default_storage_engine =@my_storage_engine;
1093-
set global thread_cache_size =@my_thread_cache_size;
10941091
set global max_allowed_packet =@my_max_allowed_packet;
10951092
set global join_buffer_size =@my_join_buffer_size;
10961093
show global variables where Variable_name='table_definition_cache';

‎mysql-test/suite/sys_vars/t/bind_address_basic.test

+2
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,8 @@
88
# Data Type: filename #
99
###############################################################################
1010

11+
--source include/not_embedded.inc
12+
1113
--echo '#---------------------BS_STVARS_001_01----------------------#'
1214
####################################################################
1315
# Displaying default value #

‎mysql-test/suite/sys_vars/t/thread_cache_size_basic.test

+1
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
--source include/not_embedded.inc
12

23
#
34
# 2010-01-20 OBN - Added check of I_S values after variable value change

‎mysql-test/suite/sys_vars/t/thread_handling_basic.test

+2
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,8 @@
2222
# #
2323
###############################################################################
2424

25+
--source include/not_embedded.inc
26+
2527
--echo '#---------------------BS_STVARS_051_01----------------------#'
2628
####################################################################
2729
# Displaying default value #

‎mysql-test/t/no-threads.test

+1
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
#but needs to be kept for tests that would need MyISAM in future.
44
--source include/force_myisam_default.inc
55

6+
--source include/not_embedded.inc
67
--source include/not_threadpool.inc
78
#
89
# Test the --thread-handler=no-threads option

0 commit comments

Comments
 (0)