Bug#36307622 [back-port] Wrong result from query with WHERE integer IN (SELECT 2 EXCEPT SELECT 4)

A work-around is to set the optimizer flag to not use hash map
de-duplication for INTERSECT, EXCEPT, like so:

SET optimizer_switch="hash_set_operations=off";

With hash_set_operations enabled, however, we get too may result rows.
For the IN predicate, the set operation is computed repeatedly, with
filters pushed down to set set operation operands:

-> Filter: <in_optimizer>(c.pk,<exists>(select #2))  (cost=2.25 rows=20)
    -> Covering index scan on c using idx_c_col_datetime_key  (cost=2.25 rows=20)
    -> Select #2 (subquery in condition; dependent)
        -> Limit: 1 row(s)  (cost=2.61..2.61 rows=1)
            -> Table scan on <except temporary>  (cost=2.61..2.61 rows=1)
                -> Except materialize with deduplication  (cost=0.1..0.1 rows=1)
                    -> Filter: (<cache>(c.pk) = <ref_null_helper>(2))  (cost=0..0 rows=1)
                        -> Rows fetched before execution  (cost=0..0 rows=1)
                    -> Filter: (<cache>(c.pk) = <ref_null_helper>(4))  (cost=0..0 rows=1)
                        -> Rows fetched before execution  (cost=0..0 rows=1)

Only the row with pk==2 should pass the filters under the except node, and that's
what happens. However, on repeated execution, the hash map used to implement
the Except materialize is not re-initialized to being empty.

The patch adds reinitialization of the hash map for such cases.

Change-Id: Idf2e36f9085e36748900017a0aad420e4e476f78

Author: Dag Wanvik

mysql-test/r/query_expression-bugs.result

@@ -334,3 +334,41 @@ j
 DROP PREPARE p0;
 SET SESSION optimizer_trace = 'enabled=default';
 SET SESSION optimizer_switch = 'hash_set_operations=default';
+#
+# Bug#36307622 Wrong result from query with WHERE integer IN (SELECT 2 EXCEPT SELECT 4)
+#
+CREATE TABLE c (
+pk int NOT NULL AUTO_INCREMENT,
+col_datetime_key datetime DEFAULT NULL,
+col_varchar_key varchar(1) DEFAULT NULL,
+PRIMARY KEY (pk),
+KEY idx_c_col_datetime_key (col_datetime_key),
+KEY idx_c_col_varchar_key (col_varchar_key)
+);
+INSERT INTO c VALUES (1,'2022-10-30 08:18:58','o');
+INSERT INTO c VALUES (2,'1998-01-19 17:27:57','䋠');
+INSERT INTO c VALUES (3,'2015-09-01 16:34:18','X');
+INSERT INTO c VALUES (4,'2020-08-29 15:09:33','m');
+INSERT INTO c VALUES (5,'2018-07-01 22:36:45','d');
+INSERT INTO c VALUES (6,'2028-02-07 02:02:10','q');
+INSERT INTO c VALUES (7,'2016-02-04 17:29:46','8');
+INSERT INTO c VALUES (8,'2037-07-02 21:02:05','0');
+INSERT INTO c VALUES (9,'1970-08-03 04:25:41','旘');
+INSERT INTO c VALUES (10,'1973-07-18 13:38:35','v');
+INSERT INTO c VALUES (11,'1990-08-03 14:18:01','υ');
+INSERT INTO c VALUES (12,'2036-07-05 20:41:55','dž');
+INSERT INTO c VALUES (13,'2035-02-11 10:59:22','䩈');
+INSERT INTO c VALUES (14,'1992-10-24 00:44:20','L');
+INSERT INTO c VALUES (15,'1995-05-04 07:35:45','W');
+INSERT INTO c VALUES (16,'2027-01-15 17:09:03','η');
+INSERT INTO c VALUES (17,'1998-02-15 07:48:55','V');
+INSERT INTO c VALUES (18,'1998-02-16 17:42:54','᥋');
+INSERT INTO c VALUES (19,'1991-03-04 15:36:40','D');
+INSERT INTO c VALUES (20,'1973-06-24 15:12:44','O');
+SELECT pk FROM c WHERE pk IN (SELECT 2 EXCEPT SELECT 4);
+pk
+2
+SELECT pk FROM c WHERE pk IN (SELECT 2 EXCEPT SELECT 4) ORDER BY pk;
+pk
+2
+DROP TABLE c;

mysql-test/t/query_expression-bugs.test

@@ -116,3 +116,41 @@ DROP PREPARE p0;
 
 SET SESSION optimizer_trace = 'enabled=default';
 SET SESSION optimizer_switch = 'hash_set_operations=default';
+
+--echo #
+--echo # Bug#36307622 Wrong result from query with WHERE integer IN (SELECT 2 EXCEPT SELECT 4)
+--echo #
+CREATE TABLE c (
+  pk int NOT NULL AUTO_INCREMENT,
+  col_datetime_key datetime DEFAULT NULL,
+  col_varchar_key varchar(1) DEFAULT NULL,
+  PRIMARY KEY (pk),
+  KEY idx_c_col_datetime_key (col_datetime_key),
+  KEY idx_c_col_varchar_key (col_varchar_key)
+);
+
+INSERT INTO c VALUES (1,'2022-10-30 08:18:58','o');
+INSERT INTO c VALUES (2,'1998-01-19 17:27:57','䋠');
+INSERT INTO c VALUES (3,'2015-09-01 16:34:18','X');
+INSERT INTO c VALUES (4,'2020-08-29 15:09:33','m');
+INSERT INTO c VALUES (5,'2018-07-01 22:36:45','d');
+INSERT INTO c VALUES (6,'2028-02-07 02:02:10','q');
+INSERT INTO c VALUES (7,'2016-02-04 17:29:46','8');
+INSERT INTO c VALUES (8,'2037-07-02 21:02:05','0');
+INSERT INTO c VALUES (9,'1970-08-03 04:25:41','旘');
+INSERT INTO c VALUES (10,'1973-07-18 13:38:35','v');
+INSERT INTO c VALUES (11,'1990-08-03 14:18:01','υ');
+INSERT INTO c VALUES (12,'2036-07-05 20:41:55','dž');
+INSERT INTO c VALUES (13,'2035-02-11 10:59:22','䩈');
+INSERT INTO c VALUES (14,'1992-10-24 00:44:20','L');
+INSERT INTO c VALUES (15,'1995-05-04 07:35:45','W');
+INSERT INTO c VALUES (16,'2027-01-15 17:09:03','η');
+INSERT INTO c VALUES (17,'1998-02-15 07:48:55','V');
+INSERT INTO c VALUES (18,'1998-02-16 17:42:54','᥋');
+INSERT INTO c VALUES (19,'1991-03-04 15:36:40','D');
+INSERT INTO c VALUES (20,'1973-06-24 15:12:44','O');
+
+SELECT pk FROM c WHERE pk IN (SELECT 2 EXCEPT SELECT 4);
+SELECT pk FROM c WHERE pk IN (SELECT 2 EXCEPT SELECT 4) ORDER BY pk;
+
+DROP TABLE c;

sql/iterators/composite_iterators.cc

@@ -618,6 +618,13 @@ class ImmutableStringHasher {
 
 using materialize_iterator::Operand;
 using Operands = Mem_root_array<Operand>;
+using hash_map_type = ankerl::unordered_dense::segmented_map<
+    ImmutableStringWithLength, LinkedImmutableString, ImmutableStringHasher>;
+
+void reset_hash_map(hash_map_type *hash_map) {
+  std::destroy_at(hash_map);
+  std::construct_at(hash_map);
+}
 
 /**
   Contains spill state for set operations' use of in-memory hash map.
@@ -876,17 +883,6 @@ class SpillState {
 
   void set_secondary_overflow() { m_secondary_overflow = true; }
 
-  using hash_map_type = ankerl::unordered_dense::segmented_map<
-      ImmutableStringWithLength, LinkedImmutableString, ImmutableStringHasher>;
-
-  static void reset_hash_map(hash_map_type *hash_map) {
-    hash_map->~hash_map_type();
-    auto *map = new (hash_map) hash_map_type();
-    if (map == nullptr) {
-      my_error(ER_OUTOFMEMORY, MYF(ME_FATALERROR), sizeof(hash_map_type));
-    }
-  }
-
   /// Getter, cf. comment for \c m_secondary_overflow
   bool secondary_overflow() const { return m_secondary_overflow; }
   void secondary_overflow_handling_done() {
@@ -1385,8 +1381,6 @@ class MaterializeIterator final : public TableRowIterator {
   MEM_ROOT *m_overflow_mem_root{nullptr};
   size_t m_row_size_upper_bound;
 
-  using hash_map_type = SpillState::hash_map_type;
-
   // The hash map where the rows are stored.
   std::unique_ptr<hash_map_type> m_hash_map;
 
@@ -1445,6 +1439,7 @@ class MaterializeIterator final : public TableRowIterator {
   bool process_row_hash(const Operand &operand, TABLE *t, ha_rows *stored_rows);
   bool materialize_hash_map(TABLE *t, ha_rows *stored_rows);
   bool load_HF_row_into_hash_map();
+  void init_hash_map_for_new_exec();
   friend class SpillState;
 };
 
@@ -1576,6 +1571,7 @@ bool MaterializeIterator<Profiler>::Init() {
   } else {
     table()->file->ha_index_or_rnd_end();  // @todo likely unneeded => remove
     table()->file->ha_delete_all_rows();
+    if (m_use_hash_map) init_hash_map_for_new_exec();
   }
 
   if (m_query_expression != nullptr)
@@ -1976,6 +1972,14 @@ bool MaterializeIterator<Profiler>::load_HF_row_into_hash_map() {
   return false;
 }
 
+template <typename Profiler>
+void MaterializeIterator<Profiler>::init_hash_map_for_new_exec() {
+  if (m_hash_map == nullptr) return;  // not used yet
+  reset_hash_map(m_hash_map.get());
+  m_mem_root->ClearForReuse();
+  m_rows_in_hash_map = 0;
+}
+
 /**
   Save (or restore) blob pointers in \c Field::m_blob_backup. We need to have
   two full copies of a record for comparison, so we save record[0] to record[1]