Bug#25650399: DOWNGRADE FROM MYSQL 8.0 TO 5.7 WILL CRASH SERVER

MySQL 8.0 adds a generated column to the two tables mysql.server_cost
and mysql.engine_cost.  MySQL 5.7 fails if system tables contain
generated columns.  Hence, downgrade to 5.7 will not work when cost
tables contain generated columns from 8.0.

This patch is for 5.7 and backports fix to read_cost_constants() from
8.0.  mysql_system_tables_fix.sql is changed to remove columns
should they exist.  This will make mysql_upgrade, if run during
downgrade, restore these tables to its original form.

sql/opt_costconstantcache.cc
   In order for resolving of generated column to work correctly when
   opening cost tables, lex_start() must be called.

scripts/mysql_system_tables_fix.sql
   Add SQL code to remove the columns from 8.0 should they exist.

scripts/mysql_system_tables.sql
   Specify only non-default values in insert statements for cost tables.
   This way mysql_upgrade does not give errors if extra columns exist.

mysql-test/t/opt_costmodel_downgrade.test
mysql-test/r/opt_costmodel_downgrade.result
   Test to verfiy that 5.7 does not crash should generated columns exist in
   cost tables.  Also verifies that running mysql_upgrade will remove
   the columns.

Author: Oystein Grovlen

mysql-test/r/opt_costmodel_downgrade.result

@@ -0,0 +1,55 @@
+
+Bug#25650399: DOWNGRADE FROM MYSQL 8.0 WITH WL#10128 TO 5.7
+WILL CRASH SERVER
+
+ALTER TABLE mysql.server_cost
+ADD COLUMN default_value FLOAT GENERATED ALWAYS AS (cost_name) VIRTUAL;
+ALTER TABLE mysql.engine_cost
+ADD COLUMN default_value FLOAT GENERATED ALWAYS AS (cost_name) VIRTUAL;
+FLUSH OPTIMIZER_COSTS;
+mysql.columns_priv                                 OK
+mysql.db                                           OK
+mysql.engine_cost                                  OK
+mysql.event                                        OK
+mysql.func                                         OK
+mysql.general_log                                  OK
+mysql.gtid_executed                                OK
+mysql.help_category                                OK
+mysql.help_keyword                                 OK
+mysql.help_relation                                OK
+mysql.help_topic                                   OK
+mysql.innodb_index_stats                           OK
+mysql.innodb_table_stats                           OK
+mysql.ndb_binlog_index                             OK
+mysql.plugin                                       OK
+mysql.proc                                         OK
+mysql.procs_priv                                   OK
+mysql.proxies_priv                                 OK
+mysql.server_cost                                  OK
+mysql.servers                                      OK
+mysql.slave_master_info                            OK
+mysql.slave_relay_log_info                         OK
+mysql.slave_worker_info                            OK
+mysql.slow_log                                     OK
+mysql.tables_priv                                  OK
+mysql.time_zone                                    OK
+mysql.time_zone_leap_second                        OK
+mysql.time_zone_name                               OK
+mysql.time_zone_transition                         OK
+mysql.time_zone_transition_type                    OK
+mysql.user                                         OK
+mtr.global_suppressions                            OK
+mtr.test_suppressions                              OK
+sys.sys_config                                     OK
+SELECT * FROM mysql.server_cost;
+cost_name	cost_value	last_update	comment
+disk_temptable_create_cost	NULL	#	NULL
+disk_temptable_row_cost	NULL	#	NULL
+key_compare_cost	NULL	#	NULL
+memory_temptable_create_cost	NULL	#	NULL
+memory_temptable_row_cost	NULL	#	NULL
+row_evaluate_cost	NULL	#	NULL
+SELECT * FROM mysql.engine_cost;
+engine_name	device_type	cost_name	cost_value	last_update	comment
+default	0	io_block_read_cost	NULL	#	NULL
+default	0	memory_block_read_cost	NULL	#	NULL

mysql-test/t/opt_costmodel_downgrade.test

@@ -0,0 +1,26 @@
+-- source include/no_valgrind_without_big.inc
+-- source include/mysql_upgrade_preparation.inc
+
+--echo
+--echo Bug#25650399: DOWNGRADE FROM MYSQL 8.0 WITH WL#10128 TO 5.7
+--echo               WILL CRASH SERVER
+--echo
+
+# Add columns and check that upgrade script drops them
+ALTER TABLE mysql.server_cost
+  ADD COLUMN default_value FLOAT GENERATED ALWAYS AS (cost_name) VIRTUAL;
+ALTER TABLE mysql.engine_cost
+  ADD COLUMN default_value FLOAT GENERATED ALWAYS AS (cost_name) VIRTUAL;
+
+# This will crash server without fix
+FLUSH OPTIMIZER_COSTS;
+
+--exec $MYSQL_UPGRADE --skip-verbose --force 2>&1
+
+# Mask out the content of the last_update column
+--replace_column 3 #
+SELECT * FROM mysql.server_cost;
+--replace_column 5 #
+SELECT * FROM mysql.engine_cost;
+
+--source include/mysql_upgrade_cleanup.inc

scripts/mysql_system_tables.sql

@@ -1,4 +1,4 @@
--- Copyright (c) 2007, 2016, Oracle and/or its affiliates. All rights reserved.
+-- Copyright (c) 2007, 2017, Oracle and/or its affiliates. All rights reserved.
 --
 -- This program is free software; you can redistribute it and/or modify
 -- it under the terms of the GNU General Public License as published by
@@ -226,23 +226,10 @@ CREATE TABLE IF NOT EXISTS server_cost (
   PRIMARY KEY (cost_name)
 ) ENGINE=InnoDB CHARACTER SET=utf8 COLLATE=utf8_general_ci STATS_PERSISTENT=0;
 
-INSERT IGNORE INTO server_cost VALUES
-  ("row_evaluate_cost", DEFAULT, CURRENT_TIMESTAMP, DEFAULT);
-
-INSERT IGNORE INTO server_cost VALUES
-  ("key_compare_cost", DEFAULT, CURRENT_TIMESTAMP, DEFAULT);
-
-INSERT IGNORE INTO server_cost VALUES
-  ("memory_temptable_create_cost", DEFAULT, CURRENT_TIMESTAMP, DEFAULT);
-
-INSERT IGNORE INTO server_cost VALUES
-  ("memory_temptable_row_cost", DEFAULT, CURRENT_TIMESTAMP, DEFAULT);
-
-INSERT IGNORE INTO server_cost VALUES
-  ("disk_temptable_create_cost", DEFAULT, CURRENT_TIMESTAMP, DEFAULT);
-
-INSERT IGNORE INTO server_cost VALUES
-  ("disk_temptable_row_cost", DEFAULT, CURRENT_TIMESTAMP, DEFAULT);
+INSERT IGNORE INTO server_cost(cost_name) VALUES
+  ("row_evaluate_cost"), ("key_compare_cost"),
+  ("memory_temptable_create_cost"), ("memory_temptable_row_cost"),
+  ("disk_temptable_create_cost"), ("disk_temptable_row_cost");
 
 -- Engine cost constants
 
@@ -256,10 +243,9 @@ CREATE TABLE IF NOT EXISTS engine_cost (
   PRIMARY KEY (cost_name, engine_name, device_type)
 ) ENGINE=InnoDB CHARACTER SET=utf8 COLLATE=utf8_general_ci STATS_PERSISTENT=0;
 
-INSERT IGNORE INTO engine_cost VALUES
-  ("default", 0, "memory_block_read_cost", DEFAULT, CURRENT_TIMESTAMP, DEFAULT);
-INSERT IGNORE INTO engine_cost VALUES
-  ("default", 0, "io_block_read_cost", DEFAULT, CURRENT_TIMESTAMP, DEFAULT);
+INSERT IGNORE INTO engine_cost(engine_name, device_type, cost_name) VALUES
+  ("default", 0, "memory_block_read_cost"),
+  ("default", 0, "io_block_read_cost");
 
 --
 -- PERFORMANCE SCHEMA INSTALLATION

scripts/mysql_system_tables_fix.sql

@@ -1,4 +1,4 @@
--- Copyright (c) 2003, 2016, Oracle and/or its affiliates. All rights reserved.
+-- Copyright (c) 2003, 2017, Oracle and/or its affiliates. All rights reserved.
 --
 -- This program is free software; you can redistribute it and/or modify
 -- it under the terms of the GNU General Public License as published by
@@ -939,3 +939,30 @@ SET @str = IF(@had_distributed_proxies_priv > 0, @cmd, "SET @dummy = 0");
 PREPARE stmt FROM @str;
 EXECUTE stmt;
 DROP PREPARE stmt;
+
+--
+-- MySQL 8.0 adds default_value column to cost tables
+-- In case of downgrade to 5.7, remove these columns
+--
+
+-- Drop column default_value from mysql.server_cost if it exists
+SET @have_server_cost_default =
+  (SELECT COUNT(column_name) FROM information_schema.columns
+     WHERE table_schema = 'mysql' AND table_name = 'server_cost' AND
+           column_name = 'default_value');
+SET @cmd="ALTER TABLE mysql.server_cost DROP COLUMN default_value";
+SET @str = IF(@have_server_cost_default > 0, @cmd, "SET @dummy = 0");
+PREPARE stmt FROM @str;
+EXECUTE stmt;
+DROP PREPARE stmt;
+
+-- Drop column default_value from mysql.engine_cost if it exists
+SET @have_engine_cost_default =
+  (SELECT COUNT(column_name) FROM information_schema.columns
+     WHERE table_schema = 'mysql' AND table_name = 'engine_cost' AND
+           column_name = 'default_value');
+SET @cmd="ALTER TABLE mysql.engine_cost DROP COLUMN default_value";
+SET @str = IF(@have_engine_cost_default > 0, @cmd, "SET @dummy = 0");
+PREPARE stmt FROM @str;
+EXECUTE stmt;
+DROP PREPARE stmt;

sql/opt_costconstantcache.cc

@@ -1,5 +1,5 @@
 /*
-   Copyright (c) 2014, 2015, Oracle and/or its affiliates. All rights reserved.
+   Copyright (c) 2014, 2017, Oracle and/or its affiliates. All rights reserved.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -28,6 +28,7 @@
 #include "records.h"                            // READ_RECORD
 #include "sql_base.h"                           // open_and_lock_tables
 #include "sql_class.h"                          // THD
+#include "sql_lex.h"                      // lex_start/lex_end
 #include "sql_string.h"                         // String
 #include "table.h"                              // TABLE
 #include "thr_lock.h"                           // TL_READ
@@ -434,6 +435,7 @@ static void read_cost_constants(Cost_model_constants* cost_constants)
   DBUG_ASSERT(thd);
   thd->thread_stack= pointer_cast<char*>(&thd);
   thd->store_globals();
+  lex_start(thd);
 
   TABLE_LIST tables[2];
   tables[0].init_one_table(C_STRING_WITH_LEN("mysql"),
@@ -462,6 +464,7 @@ static void read_cost_constants(Cost_model_constants* cost_constants)
 
   trans_commit_stmt(thd);
   close_thread_tables(thd);
+  lex_end(thd->lex);
 
   // Delete the locally created THD
   delete thd;