Bug#29634540 "DROP DATABASE OF 1 MILLION TABLES RESULTED CRASH OF MYSQLD" [noclose]

Attempt to drop database with 1 million tables results in
out-of-memory condition.

With transition to New Data-Dictionary and atomic DDL, DROP DATABASE
started to allocate much more memory which is released at the end of
statement. This memory allocated for various DD API objects, new MDL
locks, temporary objects used during updates to DD tables and so on.

Part of the problem was that DROP DATABASE for each table, tablespace,
routine or event it has dropped allocated full-blown copy of respective
DD object and stored it in thread's Dictionary_client registry of
dropped objects. These objects were kept in this registry until
statement's end. Since each individual object contained copy of
all sub-objects of the original object (even though this information
as irrelevant for tracking dropped objects) they hogged quite some
memory in total (in my model test that has dropped database with
5K of tables it was responsible for around 25% of memory consumed
by server).

This patch addresses the issue by replacing use of full-blown copies
of DD objects in thread's dropped objects registry with special
placeholders. These placeholders are almost-empty versions of
original objects with only their key members set, which is the only
info relevant for dropped objects registry.

This change doesn't solve the original problem to the full extent
but is a big step in this direction.

Reviewed-by: Dyre Tjeldvoll <Dyre.Tjeldvoll@oracle.com>

Author: Dmitry Lenev

mysql-test/r/dd_pfs.result

@@ -35,16 +35,16 @@ include/assert.inc ['SELECT * FROM s.t_innodb_1' will allocate 0 objects.]
 include/assert.inc ['SELECT * FROM s.t_innodb_1' will free 0 objects.]
 include/assert.inc ['SELECT * FROM s.t_innodb_1' will leave 0 objects.]
 ###############################################################################
-include/assert.inc ['DROP TABLE s.t_innodb_1' will allocate 16 objects.]
-include/assert.inc ['DROP TABLE s.t_innodb_1' will free 26 objects.]
+include/assert.inc ['DROP TABLE s.t_innodb_1' will allocate 7 objects.]
+include/assert.inc ['DROP TABLE s.t_innodb_1' will free 17 objects.]
 include/assert.inc ['DROP TABLE s.t_innodb_1' will leave -10 objects.]
 ###############################################################################
-include/assert.inc ['DROP TABLE s.t_innodb_2' will allocate 16 objects.]
-include/assert.inc ['DROP TABLE s.t_innodb_2' will free 26 objects.]
+include/assert.inc ['DROP TABLE s.t_innodb_2' will allocate 7 objects.]
+include/assert.inc ['DROP TABLE s.t_innodb_2' will free 17 objects.]
 include/assert.inc ['DROP TABLE s.t_innodb_2' will leave -10 objects.]
 ###############################################################################
-include/assert.inc ['DROP SCHEMA s' will allocate 3 objects.]
-include/assert.inc ['DROP SCHEMA s' will free 6 objects.]
+include/assert.inc ['DROP SCHEMA s' will allocate 2 objects.]
+include/assert.inc ['DROP SCHEMA s' will free 5 objects.]
 include/assert.inc ['DROP SCHEMA s' will leave -3 objects.]
 ###############################################################################
 include/assert.inc ['SELECT * FROM test.tables' will allocate 168 objects.]
@@ -55,5 +55,6 @@ We will see that the total size allocated is about 40K.
 include/assert.inc ['' will allocate 151 + 151 objects.]
 include/assert.inc ['' will free 151 + 151 objects.]
 ###############################################################################
-include/assert.inc ['DROP SCHEMA s' will allocate 3801 objects.]
-include/assert.inc ['DROP SCHEMA s' will free 3802 objects.]
+include/assert.inc ['DROP SCHEMA s' will allocate 601 objects.]
+include/assert.inc ['DROP SCHEMA s' will free 2202 objects.]
+include/assert.inc ['DROP SCHEMA s' will leave -1601 objects.]

mysql-test/t/dd_pfs.test

@@ -147,13 +147,14 @@ let $operation= SELECT * FROM s.t_innodb_1;
 # We first fill in tablespace names referred by the table and index, allocating
 # tablespace and file objects uncached twice (+4). These are then freed (-4).
 # While updating the SDI, the tablespace and file objects are allocated uncached
-# again (+2). Then, the table object(s) are cloned (+10) and kept as uncommitted
-# objects, and the cached objects are freed (-10). Finally, the uncommitted
-# objects are freed (-10), and the remaining uncached ones are deleted (-2). So
-# this results in 16 allocations and 26 frees, and a net change of -10.
-
-let $object_alloc= 16;
-let $object_free= 26;
+# again (+2). Then, the placeholder object is cloned from the table object (+1)
+# and kept as uncommitted object, and the cached table object and its subobjects
+# are freed (-10). Finally, the uncommitted object is freed (-1), and the
+# remaining uncached ones are deleted (-2). So this results in 7 allocations and
+# 17 frees, and a net change of -10.
+
+let $object_alloc= 7;
+let $object_free= 17;
 let $object_count= -10;
 let $operation= DROP TABLE s.t_innodb_1;
 --source include/dd_pfs_report_state.inc
@@ -163,22 +164,22 @@ let $operation= DROP TABLE s.t_innodb_1;
 #
 # We get the same scenario as above.
 
-let $object_alloc= 16;
-let $object_free= 26;
+let $object_alloc= 7;
+let $object_free= 17;
 let $object_count= -10;
 let $operation= DROP TABLE s.t_innodb_2;
 --source include/dd_pfs_report_state.inc
 
 ###############################################################################
 # Drop the schema.
 #
-# We will clone the table object (+2) and the schema object (+1) and store them
-# as uncommitted objects. Then, the objects will be freed from the DD cache (-3).
-# Finally, the uncommitted objects are freed (-3). This gives us 3 allocations,
-# 6 frees and a net change of -3 objects.
+# We will create placeholders for the table object (+1) and the schema object (+1)
+# and store them as uncommitted objects. Then, the objects will be freed from the DD
+# cache (-3). Finally, the uncommitted objects are freed (-2). This gives us 2
+# allocations, 5 frees and a net change of -3 objects.
 
-let $object_alloc= 3;
-let $object_free= 6;
+let $object_alloc= 2;
+let $object_free= 5;
 let $object_count= -3;
 let $operation= DROP SCHEMA s;
 --source include/dd_pfs_report_state.inc
@@ -335,11 +336,13 @@ while ($i > 0)
 --enable_query_log
 
 ###############################################################################
-# Mem usage by drop schema
+# Mem usage by drop schema / test for partial fix for bug#29634540
+# "DROP DATABASE OF 1 MILLION TABLES RESULTED CRASH OF MYSQLD".
 #
-# Dropping a schema will hog memory because the dictionary client will store
-# all DD objects for the tables in the schema in memory while the schema is
-# being deleted. The DROP SCHEMA below will allocate about 2MB of memory.
+# Dropping a schema was hogging memory because the dictionary client was
+# storingfull DD objects for all tables in the schema in memory while the
+# schema was deleted. After partial fix for the problem we now store slimmed
+# down placeholders instead of full-blown DD objects for all tables instead.
 
 --disable_query_log
 --disable_result_log
@@ -349,20 +352,43 @@ while ($i > 0)
 {
   dec $i;
   eval CREATE TABLE s.t_$i (i int, j int, k int, l int);
+  # Pull in DD objects for tables in cache.
+  eval SELECT * FROM s.t_$i;
 }
 --enable_result_log
 --enable_query_log
 
+# For each table:
+#
+#   We first get tablespace name referred by the index in order to acquire
+#   MDL on it, allocating tablespace and file objects uncached (+2) and
+#   then freeing them immediately (-2).
+#
+#   Then during call to drop table in SE, InnoDB will drop file-per-table
+#   tablespace. To do this it will get get uncached tablespace and file
+#   objects once again (+2). The placeholder for tablespace object will
+#   be created and added to uncommited registry (+1), Original uncached
+#   tablespace and file objects will be deleted (-2).
+#
+#   After that we will delete table from DD, and in the process we will
+#   create placeholder for table object (+1) and keep it as uncommitted,
+#   free cached table object and its subobjects (-16).
+#
+# Then once for the statement, we will create placeholder for schema
+# object to keep as uncommited (+1) and free cached schema object (-1).
+#
+# Finally, we will free uncommitted placeholders for each table, tablespace
+# (-2 per table) and for schema (-1).
+#
+# So in total this results in (2 + 2 + 1 + 1) * 100 + 1 = 601 allocations,
+# (2 + 2 + 16) * 100 + 1 + 2 * 100 + 1 = 2202 frees, and a net change of -1601.
 --source include/dd_pfs_save_state.inc
-let $object_alloc= 3801;
-let $object_free= 3802;
-let $object_left= -1;
+let $object_alloc= 601;
+let $object_free= 2202;
+let $object_count= -1601;
 let $operation= DROP SCHEMA s;
 --source include/dd_pfs_report_state.inc
 
-# Using the DD P_S instrumentation, it will at least be possible to monitor
-# the memory usage.
-
 ###############################################################################
 # Cleanup.
 

sql/dd/impl/cache/dictionary_client.cc

@@ -2468,11 +2468,14 @@ bool Dictionary_client::drop(const T *object) {
     return true;
   }
 
-  // Prepare an instance to be added to the dropped registry. This must be done
-  // prior to cleaning up the committed registry since the instance we drop
-  // might be present there (since we are allowed to drop const object coming
-  // from acquire()).
-  T *dropped_object = object->clone();
+  // Prepare an object placeholder to be added to the dropped registry.
+  // This must be done prior to cleaning up the committed registry since
+  // the instance we drop might be present there (since we are allowed to
+  // drop const object coming from acquire()).
+  // We use placeholder instead of simple clone of the original object in
+  // order to avoid consuming too much memory in cases when we need to drop
+  // many (thousands or more) objects within the single atomic operation.
+  T *dropped_object = object->clone_dropped_object_placeholder();
 
   // Invalidate the entry in the shared cache (if present).
   invalidate(object);

sql/dd/impl/types/charset_impl.h

@@ -127,6 +127,17 @@ class Charset_impl : public Entity_object_impl, public Charset {
   Object_id m_default_collation_id;
 
   Charset *clone() const override { return new Charset_impl(*this); }
+
+  Charset *clone_dropped_object_placeholder() const override {
+    /*
+      Even though we don't drop charsets en masse we still create slimmed
+      down version for consistency sake.
+    */
+    Charset_impl *placeholder = new Charset_impl();
+    placeholder->set_id(id());
+    placeholder->set_name(name());
+    return placeholder;
+  }
 };
 
 ///////////////////////////////////////////////////////////////////////////

sql/dd/impl/types/collation_impl.h

@@ -137,6 +137,11 @@ class Collation_impl : public Entity_object_impl, public Collation {
   Object_id m_charset_id;
 
   Collation *clone() const override { return new Collation_impl(*this); }
+
+  Collation *clone_dropped_object_placeholder() const override {
+    // Play simple. Proper placeholder will take the same memory as clone.
+    return clone();
+  }
 };
 
 ///////////////////////////////////////////////////////////////////////////

sql/dd/impl/types/column_statistics_impl.h

@@ -160,6 +160,21 @@ class Column_statistics_impl final : public Entity_object_impl,
   Column_statistics *clone() const override {
     return new Column_statistics_impl(*this);
   }
+
+  Column_statistics *clone_dropped_object_placeholder() const override {
+    Column_statistics_impl *placeholder = new Column_statistics_impl();
+    placeholder->set_id(id());
+    placeholder->set_name(name());
+    /*
+      Even though schema, table and column name members are not used in keys
+      directly, they are still used to check that correct metadata locks are
+      held, so for safety we copy them as well.
+    */
+    placeholder->set_schema_name(schema_name());
+    placeholder->set_table_name(table_name());
+    placeholder->set_column_name(column_name());
+    return placeholder;
+  }
 };
 
 ///////////////////////////////////////////////////////////////////////////

sql/dd/impl/types/event_impl.h

@@ -370,6 +370,14 @@ class Event_impl : public Entity_object_impl, public Event {
   Object_id m_schema_collation_id;
 
   Event *clone() const override { return new Event_impl(*this); }
+
+  Event *clone_dropped_object_placeholder() const override {
+    Event_impl *placeholder = new Event_impl();
+    placeholder->set_id(id());
+    placeholder->set_schema_id(schema_id());
+    placeholder->set_name(name());
+    return placeholder;
+  }
 };
 
 ///////////////////////////////////////////////////////////////////////////

sql/dd/impl/types/function_impl.h

@@ -340,6 +340,16 @@ class Function_impl : public Routine_impl, public Function {
 
   Function_impl(const Function_impl &src);
   Function_impl *clone() const override { return new Function_impl(*this); }
+
+  // N.B.: returning dd::Function from this function confuses MSVC compiler
+  // thanks to diamond inheritance.
+  Function_impl *clone_dropped_object_placeholder() const override {
+    Function_impl *placeholder = new Function_impl();
+    placeholder->set_id(id());
+    placeholder->set_schema_id(schema_id());
+    placeholder->set_name(name());
+    return placeholder;
+  }
 };
 
 ///////////////////////////////////////////////////////////////////////////

sql/dd/impl/types/procedure_impl.h

@@ -174,6 +174,16 @@ class Procedure_impl : public Routine_impl, public Procedure {
  private:
   Procedure_impl(const Procedure_impl &src);
   Procedure_impl *clone() const override { return new Procedure_impl(*this); }
+
+  // N.B.: returning dd::Procedure from this function might confuse MSVC
+  // compiler thanks to diamond inheritance.
+  Procedure_impl *clone_dropped_object_placeholder() const override {
+    Procedure_impl *placeholder = new Procedure_impl();
+    placeholder->set_id(id());
+    placeholder->set_schema_id(schema_id());
+    placeholder->set_name(name());
+    return placeholder;
+  }
 };
 
 ///////////////////////////////////////////////////////////////////////////

sql/dd/impl/types/resource_group_impl.h

@@ -113,6 +113,11 @@ class Resource_group_impl : public Entity_object_impl, public Resource_group {
   Resource_group *clone() const override {
     return new Resource_group_impl(*this);
   }
+
+  Resource_group *clone_dropped_object_placeholder() const override {
+    // Play simple. Proper placeholder will take the same memory as clone.
+    return clone();
+  }
 };
 
 }  // namespace dd

sql/dd/impl/types/schema_impl.h

@@ -231,6 +231,17 @@ class Schema_impl : public Entity_object_impl, public Schema {
   Object_id m_default_collation_id;
 
   Schema *clone() const override { return new Schema_impl(*this); }
+
+  Schema *clone_dropped_object_placeholder() const override {
+    /*
+      Even though we don't drop databases en masse we still create slimmed
+      down version for consistency sake.
+    */
+    Schema_impl *placeholder = new Schema_impl();
+    placeholder->set_id(id());
+    placeholder->set_name(name());
+    return placeholder;
+  }
 };
 
 ///////////////////////////////////////////////////////////////////////////

sql/dd/impl/types/spatial_reference_system_impl.h

@@ -366,6 +366,18 @@ class Spatial_reference_system_impl : public Entity_object_impl,
   Spatial_reference_system *clone() const override {
     return new Spatial_reference_system_impl(*this);
   }
+
+  Spatial_reference_system *clone_dropped_object_placeholder() const override {
+    /*
+      Even though we don't drop SRSes en masse we still create slimmed
+      down version for consistency sake.
+    */
+    Spatial_reference_system_impl *placeholder =
+        new Spatial_reference_system_impl();
+    placeholder->set_id(id());
+    placeholder->set_name(name());
+    return placeholder;
+  }
 };
 
 ///////////////////////////////////////////////////////////////////////////

sql/dd/impl/types/table_impl.h

@@ -591,6 +591,25 @@ class Table_impl : public Abstract_table_impl, virtual public Table {
 
   Table_impl(const Table_impl &src);
   Table_impl *clone() const override { return new Table_impl(*this); }
+
+  // N.B.: returning dd::Table from this function might confuse MSVC
+  // compiler thanks to diamond inheritance.
+  Table_impl *clone_dropped_object_placeholder() const override {
+    /*
+      TODO: In future we might want to save even more memory and use separate
+            placeholder class implementing dd::Table interface instead of
+            Table_impl. Instances of such class can be several times smaller
+            than an empty Table_impl. It might make sense to do the same for
+            for some of other types as well.
+    */
+    Table_impl *placeholder = new Table_impl();
+    placeholder->set_id(id());
+    placeholder->set_schema_id(schema_id());
+    placeholder->set_name(name());
+    placeholder->set_engine(engine());
+    placeholder->set_se_private_id(se_private_id());
+    return placeholder;
+  }
 };
 
 ///////////////////////////////////////////////////////////////////////////

sql/dd/impl/types/tablespace_impl.h

@@ -176,6 +176,13 @@ class Tablespace_impl : public Entity_object_impl, public Tablespace {
   Tablespace_impl(const Tablespace_impl &src);
 
   Tablespace *clone() const override { return new Tablespace_impl(*this); }
+
+  Tablespace *clone_dropped_object_placeholder() const override {
+    Tablespace_impl *placeholder = new Tablespace_impl();
+    placeholder->set_id(id());
+    placeholder->set_name(name());
+    return placeholder;
+  }
 };
 
 ///////////////////////////////////////////////////////////////////////////

sql/dd/impl/types/view_impl.h

@@ -300,6 +300,16 @@ class View_impl : public Abstract_table_impl, public View {
 
   View_impl(const View_impl &src);
   View_impl *clone() const override { return new View_impl(*this); }
+
+  // N.B.: returning dd::View from this function confuses MSVC compiler
+  // thanks to diamond inheritance.
+  View_impl *clone_dropped_object_placeholder() const override {
+    View_impl *placeholder = new View_impl();
+    placeholder->set_id(id());
+    placeholder->set_schema_id(schema_id());
+    placeholder->set_name(name());
+    return placeholder;
+  }
 };
 
 ///////////////////////////////////////////////////////////////////////////

sql/dd/types/abstract_table.h

@@ -180,6 +180,14 @@ class Abstract_table : virtual public Entity_object {
     @return pointer to dynamically allocated copy
   */
   virtual Abstract_table *clone() const = 0;
+
+  /**
+    Allocate a new object which can serve as a placeholder for the original
+    object in the Dictionary_client's dropped registry. Such object has the
+    same keys as the original but has no other info and as result occupies
+    less memory.
+  */
+  virtual Abstract_table *clone_dropped_object_placeholder() const = 0;
 };
 
 ///////////////////////////////////////////////////////////////////////////