Bug #21650559 MYSQLPUMP MAKES CORRUPTED DUMP WHEN TABLE HAS GENERATED COLUMN

Problem: mysqlpump does not generate correct INSERT sql statements in dump
file for tables which have generated columns. This will lead to error during
restore.

Fix: For tables with generated columns mysqlpump first identifies all the
generated columns before preparing the INSERT sql statement for any table.
Then those generated columns are ignored in the column name list of INSERT
sql so that these sql statements get executed correctly during restore.

Author: Bharathy Satish

client/dump/mysql_field.cc

@@ -18,26 +18,32 @@
 #include "mysql_field.h"
 
 using namespace Mysql::Tools::Dump;
+using namespace std;
 
 enum enum_field_types Mysql_field::get_type() const
 {
-  return m_field.type;
+  return m_type;
 }
 
 uint Mysql_field::get_additional_flags() const
 {
-  return m_field.flags;
+  return m_flags;
 }
 
 uint Mysql_field::get_character_set_nr() const
 {
-  return m_field.charsetnr;
+  return m_charsetnr;
 }
 
 std::string Mysql_field::get_name() const
 {
-  return m_field.name;
+  return m_name;
 }
 
-Mysql_field::Mysql_field(MYSQL_FIELD* field) : m_field(*field)
-{}
+Mysql_field::Mysql_field(MYSQL_FIELD* field)
+  : m_name(field->name),
+    m_charsetnr(field->charsetnr),
+    m_flags(field->flags),
+    m_type(field->type)
+{
+}

client/dump/mysql_field.h

@@ -40,7 +40,10 @@ class Mysql_field
   enum enum_field_types get_type() const;
 
 private:
-  MYSQL_FIELD m_field;
+  std::string m_name;
+  unsigned int m_charsetnr;
+  unsigned int m_flags;
+  enum enum_field_types m_type;
 };
 
 }

client/dump/mysql_object_reader.cc

@@ -16,6 +16,7 @@
 */
 
 #include "mysql_object_reader.h"
+#include <boost/algorithm/string.hpp>
 
 using namespace Mysql::Tools::Dump;
 
@@ -24,6 +25,7 @@ void Mysql_object_reader::Rows_fetching_context::acquire_fields_information(
 {
   MYSQL_FIELD* fields= mysql_fetch_fields(mysql_result);
   uint field_count= mysql_num_fields(mysql_result);
+  m_fields.reserve(field_count);
   for (uint i= 0; i < field_count; ++i)
     m_fields.push_back(Mysql_field(&fields[i]));
 }
@@ -56,11 +58,13 @@ int64 Mysql_object_reader::Rows_fetching_context::result_callback(
 }
 
 Mysql_object_reader::Rows_fetching_context::Rows_fetching_context(
-  Mysql_object_reader* parent, Item_processing_data* item_processing)
+  Mysql_object_reader* parent, Item_processing_data* item_processing,
+  bool has_generated_column)
   : m_parent(parent),
   m_item_processing(item_processing),
   m_row_group((Table*)item_processing
-    ->get_process_task_object()->get_related_db_object(), m_fields)
+    ->get_process_task_object()->get_related_db_object(), m_fields,
+    has_generated_column)
 {
   m_row_group.m_rows.reserve(
     (size_t)m_parent->m_options->m_row_group_size);
@@ -75,18 +79,44 @@ void Mysql_object_reader::read_table_rows_task(
   Table_rows_dump_task* table_rows_dump_task,
   Item_processing_data* item_to_process)
 {
+  bool has_generated_columns= 0;
   Mysql::Tools::Base::Mysql_query_runner* runner= this->get_runner();
   Table* table= table_rows_dump_task->get_related_table();
 
+  std::vector<const Mysql::Tools::Base::Mysql_query_runner::Row*> columns;
+  std::vector<std::string> field_names;
+
+  runner->run_query_store(
+    "SELECT `COLUMN_NAME`, `EXTRA` FROM " +
+    this->get_quoted_object_full_name("INFORMATION_SCHEMA", "COLUMNS") +
+    "WHERE TABLE_SCHEMA ='" + runner->escape_string(table->get_schema()) +
+    "' AND TABLE_NAME ='" + runner->escape_string(table->get_name()) + "'",
+    &columns);
+
+  std::string column_names;
+  for (std::vector<const Mysql::Tools::Base::Mysql_query_runner::Row*>::iterator
+    it= columns.begin(); it != columns.end(); ++it)
+  {
+    const Mysql::Tools::Base::Mysql_query_runner::Row& column_data= **it;
+    if (column_data[1] == "STORED GENERATED" ||
+        column_data[1] == "VIRTUAL GENERATED")
+      has_generated_columns= 1;
+    else
+      column_names+= this->quote_name(column_data[0]) + ",";
+  }
+  /* remove last comma from column_names */
+  column_names= boost::algorithm::replace_last_copy(column_names, ",", "");
+
   Rows_fetching_context* row_fetching_context=
-    new Rows_fetching_context(this, item_to_process);
+    new Rows_fetching_context(this, item_to_process, has_generated_columns);
 
   runner->run_query(
-    "SELECT SQL_NO_CACHE * FROM " + this->get_quoted_object_full_name(table),
+    "SELECT SQL_NO_CACHE " + column_names + "  FROM " +
+    this->get_quoted_object_full_name(table),
     new Mysql::Instance_callback<
       int64, const Mysql::Tools::Base::Mysql_query_runner::Row&,
-      Rows_fetching_context>(
-        row_fetching_context, &Rows_fetching_context::result_callback));
+        Rows_fetching_context>(
+          row_fetching_context, &Rows_fetching_context::result_callback));
 
   row_fetching_context->process_buffer();
   if (row_fetching_context->is_all_rows_processed())

client/dump/mysql_object_reader.h

@@ -60,7 +60,7 @@ class Mysql_object_reader
   {
   public:
     Rows_fetching_context(Mysql_object_reader* parent,
-        Item_processing_data* item_processing);
+        Item_processing_data* item_processing, bool has_generated_column);
 
     int64 result_callback(
       const Mysql::Tools::Base::Mysql_query_runner::Row& row_data);

client/dump/row_group_dump_task.cc

@@ -42,7 +42,9 @@ I_data_object* Row_group_dump_task::get_related_db_object() const
 }
 
 Row_group_dump_task::Row_group_dump_task(Table* source_table,
-  const std::vector<Mysql_field>& fields)
+  const std::vector<Mysql_field>& fields,
+  const bool has_generated_column)
   : m_source_table(source_table),
-  m_fields(fields)
+  m_fields(fields),
+  m_has_generated_columns(has_generated_column)
 {}

client/dump/row_group_dump_task.h

@@ -35,7 +35,8 @@ class Row_group_dump_task : public Abstract_simple_dump_task
 {
 public:
   Row_group_dump_task(
-      Table* source_table, const std::vector<Mysql_field>& fields);
+      Table* source_table, const std::vector<Mysql_field>& fields,
+      const bool has_generated_column);
 
   virtual I_data_object* get_related_db_object() const;
 
@@ -46,7 +47,7 @@ class Row_group_dump_task : public Abstract_simple_dump_task
   /**
     Returns a table the rows are contained in.
    */
-  Table* m_source_table;
+  const Table* m_source_table;
   /**
     Contains all fields information.
    */
@@ -55,6 +56,10 @@ class Row_group_dump_task : public Abstract_simple_dump_task
     Returns all rows.
    */
   std::vector<Row*> m_rows;
+  /**
+    Contains generated/virtual fields.
+  */
+  const bool m_has_generated_columns;
 };
 
 }

client/dump/sql_formatter.cc

@@ -42,14 +42,13 @@ void Sql_formatter::format_row_group(Row_group_dump_task* row_group)
       row_data_length+= row->m_row_data.size_of_element(column) * 2 + 3;
     }
   }
-  if (m_options->m_dump_column_names)
+  if (m_options->m_dump_column_names || row_group->m_has_generated_columns)
   {
     row_data_length+= 3; // Space for enclosing parentheses and space.
-    const std::vector<Field>& fields=
-      row_group->m_source_table->get_fields();
-    for (std::vector<Field>::const_iterator field_iterator= fields.begin();
-      field_iterator != fields.end();
-      ++field_iterator)
+    const std::vector<Mysql_field>& fields= row_group->m_fields;
+    for (std::vector<Mysql_field>::const_iterator
+       field_iterator= fields.begin(); field_iterator != fields.end();
+       ++field_iterator)
     {
       row_data_length+= field_iterator->get_name().size() * 2 + 3;
     }
@@ -73,13 +72,12 @@ void Sql_formatter::format_row_group(Row_group_dump_task* row_group)
   else
     row_string+= "INSERT INTO ";
   row_string+= this->get_quoted_object_full_name(row_group->m_source_table);
-  if (m_options->m_dump_column_names)
+  if (m_options->m_dump_column_names || row_group->m_has_generated_columns)
   {
     row_string+= " (";
-    const std::vector<Field>& fields=
-      row_group->m_source_table->get_fields();
-    for (std::vector<Field>::const_iterator field_iterator= fields.begin();
-      field_iterator != fields.end();
+    const std::vector<Mysql_field>& fields= row_group->m_fields;
+    for (std::vector<Mysql_field>::const_iterator
+      field_iterator= fields.begin(); field_iterator != fields.end();
       ++field_iterator)
     {
       if (field_iterator != fields.begin())
@@ -143,8 +141,7 @@ void Sql_formatter::format_row_group(Row_group_dump_task* row_group)
         {
           row_string+= "NULL";
         }
-        else if (row_group->m_fields[column].get_type()
-          == MYSQL_TYPE_DECIMAL)
+        else if (row_group->m_fields[column].get_type() == MYSQL_TYPE_DECIMAL)
         {
           row_string+= '\'';
           row_string.append(column_data, column_length);

client/dump/thread_specific_connection_provider.cc

@@ -29,6 +29,11 @@ Mysql::Tools::Base::Mysql_query_runner*
   {
     runner= this->create_new_runner(message_handler);
     runner->run_query("SET SQL_QUOTE_SHOW_CREATE= 1");
+    /*
+      Do not allow server to make any timezone converstion even if it
+      has a different time zone set.
+    */
+    runner->run_query("SET TIME_ZONE='+00:00'");
     m_runner.reset(runner);
   }
   // Deliver copy of original runner.
@@ -38,4 +43,4 @@ Mysql::Tools::Base::Mysql_query_runner*
 Thread_specific_connection_provider::Thread_specific_connection_provider(
   Mysql::Tools::Base::I_connection_factory* connection_factory)
   : Abstract_connection_provider(connection_factory)
-{}
+{}

mysql-test/r/mysqlpump_basic.result

@@ -204,16 +204,22 @@ trig1
 trig2
 SELECT 1 FROM db1_basic.t4 WHERE name= 'disk_temptable_create_cost' AND last_update = @val1;
 1
+1
 SELECT 2 FROM db1_basic.t4 WHERE name= 'disk_temptable_row_cost' AND last_update = @val2;
 2
+2
 SELECT 3 FROM db1_basic.t4 WHERE name= 'key_compare_cost' AND last_update = @val3;
 3
+3
 SELECT 4 FROM db1_basic.t4 WHERE name= 'memory_temptable_create_cost' AND last_update = @val4;
 4
+4
 SELECT 5 FROM db1_basic.t4 WHERE name= 'memory_temptable_row_cost' AND last_update = @val5;
 5
+5
 SELECT 6 FROM db1_basic.t4 WHERE name= 'row_evaluate_cost' AND last_update = @val6;
 6
+6
 # test for skip-triggers
 DROP DATABASE db1_basic;
 SHOW TRIGGERS FROM db1_basic;
@@ -495,3 +501,53 @@ disk_temptable_row_cost
 SELECT * FROM t5 ORDER BY 1;
 id	id2
 DROP DATABASE bug21644479;
+#
+# Bug #21650559 MYSQLPUMP MAKES CORRUPTED DUMP WHEN TABLE HAS GENERATED COLUMN
+#
+CREATE DATABASE bug21650559;
+USE bug21650559;
+CREATE TABLE t1 (pk INTEGER, a INTEGER, b INTEGER, c VARCHAR(16),
+sum INTEGER GENERATED ALWAYS AS (a+b),
+sub VARCHAR(4) GENERATED ALWAYS AS (SUBSTRING(c, 1, 4)),
+key k1(sum),
+key k2(sub)
+);
+CREATE TABLE t2 (pk INTEGER, a INTEGER, b INTEGER,
+sum INTEGER GENERATED ALWAYS AS (a+b),
+c VARCHAR(16),
+key k1(sum)
+);
+CREATE TABLE t3 (sum INTEGER GENERATED ALWAYS AS (a+(EXTRACT(YEAR from pk))),
+pk TIMESTAMP, a INTEGER,
+c VARCHAR(16)
+);
+INSERT INTO t1(pk, a, b, c) VALUES (1, 11, 12, 'oneone'), (2, 21, 22, 'twotwo');
+SELECT * FROM t1;
+pk	a	b	c	sum	sub
+1	11	12	oneone	23	oneo
+2	21	22	twotwo	43	twot
+INSERT INTO t2(pk, a, b, c) VALUES (1, 11, 12, 'oneone'), (2, 21, 22, 'twotwo');
+SELECT * FROM t2;
+pk	a	b	sum	c
+1	11	12	23	oneone
+2	21	22	43	twotwo
+INSERT INTO t3(pk, a, c) VALUES ('2003-01-02 10:30:00.000123', 11, 12), ('2015-11-22 00:30:00', 21, 22);
+SELECT * FROM t3;
+sum	pk	a	c
+2014	2003-01-02 10:30:00	11	12
+2036	2015-11-22 00:30:00	21	22
+DROP DATABASE bug21650559;
+USE bug21650559;
+SELECT * FROM t1;
+pk	a	b	c	sum	sub
+1	11	12	oneone	23	oneo
+2	21	22	twotwo	43	twot
+SELECT * FROM t2;
+pk	a	b	sum	c
+1	11	12	23	oneone
+2	21	22	43	twotwo
+SELECT * FROM t3;
+sum	pk	a	c
+2014	2003-01-02 10:30:00	11	12
+2036	2015-11-22 00:30:00	21	22
+DROP DATABASE bug21650559;

mysql-test/t/mysqlpump_basic.test

@@ -438,3 +438,42 @@ DROP DATABASE bug21644479;
 --remove_file $MYSQLTEST_VARDIR/tmp/bug21644479_zlib.sql
 --remove_file $MYSQLTEST_VARDIR/tmp/bug21644479.lz4
 --remove_file $MYSQLTEST_VARDIR/tmp/bug21644479.zlib
+
+--echo #
+--echo # Bug #21650559 MYSQLPUMP MAKES CORRUPTED DUMP WHEN TABLE HAS GENERATED COLUMN
+--echo #
+
+CREATE DATABASE bug21650559;
+USE bug21650559;
+CREATE TABLE t1 (pk INTEGER, a INTEGER, b INTEGER, c VARCHAR(16),
+                 sum INTEGER GENERATED ALWAYS AS (a+b),
+                 sub VARCHAR(4) GENERATED ALWAYS AS (SUBSTRING(c, 1, 4)),
+                 key k1(sum),
+                 key k2(sub)
+);
+CREATE TABLE t2 (pk INTEGER, a INTEGER, b INTEGER,
+                 sum INTEGER GENERATED ALWAYS AS (a+b),
+                 c VARCHAR(16),
+                 key k1(sum)
+);
+CREATE TABLE t3 (sum INTEGER GENERATED ALWAYS AS (a+(EXTRACT(YEAR from pk))),
+                 pk TIMESTAMP, a INTEGER,
+                 c VARCHAR(16)
+);
+
+INSERT INTO t1(pk, a, b, c) VALUES (1, 11, 12, 'oneone'), (2, 21, 22, 'twotwo');
+SELECT * FROM t1;
+INSERT INTO t2(pk, a, b, c) VALUES (1, 11, 12, 'oneone'), (2, 21, 22, 'twotwo');
+SELECT * FROM t2;
+INSERT INTO t3(pk, a, c) VALUES ('2003-01-02 10:30:00.000123', 11, 12), ('2015-11-22 00:30:00', 21, 22);
+SELECT * FROM t3;
+
+--exec $MYSQL_PUMP --databases bug21650559 > $MYSQLTEST_VARDIR/tmp/bug21650559.sql
+DROP DATABASE bug21650559;
+--exec $MYSQL < $MYSQLTEST_VARDIR/tmp/bug21650559.sql
+--remove_file $MYSQLTEST_VARDIR/tmp/bug21650559.sql
+USE bug21650559;
+SELECT * FROM t1;
+SELECT * FROM t2;
+SELECT * FROM t3;
+DROP DATABASE bug21650559;