ndb - merge 70-spj-scan-scan to 72

Author: Jonas Oreland

mysql-test/suite/ndb/r/ndb_join_pushdown.result

@@ -2060,33 +2060,33 @@ drop table tx;
 alter table t1 partition by key(a);
 explain select count(*) from t1 
 join t1 as t2 on t2.a = t1.c 
-join t1 as t3 on t3.a = t1.c;
+join t1 as t3 on t3.a = t1.d;
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
 1	SIMPLE	t1	ALL	NULL	NULL	NULL	NULL	16	Parent of 3 pushed join@1
 1	SIMPLE	t2	ref	PRIMARY	PRIMARY	4	test.t1.c	1	Child of 't1' in pushed join@1
-1	SIMPLE	t3	ref	PRIMARY	PRIMARY	4	test.t1.c	1	Child of 't1' in pushed join@1
+1	SIMPLE	t3	ref	PRIMARY	PRIMARY	4	test.t1.d	1	Child of 't1' in pushed join@1
 select count(*) from t1 
 join t1 as t2 on t2.a = t1.c 
-join t1 as t3 on t3.a = t1.c;
+join t1 as t3 on t3.a = t1.d;
 count(*)
-208
+176
 CREATE TABLE tx (
 a int NOT NULL,
 PRIMARY KEY (`a`)
 );
 delete from t1;
 insert into tx values (0), (1), (2), (3), (4), (5), (6), (7), (8), (9);
-insert into t1 select 1, x1.a * 10+x2.a, 1, 0 from tx as x1 cross join tx as x2;
+insert into t1 select 1, x1.a * 10+x2.a, 1, 1 from tx as x1 cross join tx as x2;
 explain select count(*) from t1 as x1
 join t1 as x2 on x2.a = x1.c and x1.b < 2 
-join t1 as x3 on x3.a = x1.c;
+join t1 as x3 on x3.a = x1.d;
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
 1	SIMPLE	x1	ALL	NULL	NULL	NULL	NULL	100	Parent of 3 pushed join@1; Using where with pushed condition
 1	SIMPLE	x2	ref	PRIMARY	PRIMARY	4	test.x1.c	1	Child of 'x1' in pushed join@1
-1	SIMPLE	x3	ref	PRIMARY	PRIMARY	4	test.x1.c	1	Child of 'x1' in pushed join@1
+1	SIMPLE	x3	ref	PRIMARY	PRIMARY	4	test.x1.d	1	Child of 'x1' in pushed join@1
 select count(*) from t1 as x1
 join t1 as x2 on x2.a = x1.c and x1.b < 2 
-join t1 as x3 on x3.a = x1.c;
+join t1 as x3 on x3.a = x1.d;
 count(*)
 20000
 drop table t1;
@@ -2925,20 +2925,22 @@ a3	b3	c3	d3	a3	b3	c3	d3
 explain extended
 select straight_join * from
 t3 as x1
-join t3 as y1 on x1.b3 = y1.b3 and x1.d3 = y1.d3
-join t3 as x2 on x2.b3 = y1.b3
+join t3 as y1 on y1.b3 = x1.b3 and y1.d3 = x1.d3
+join t3 as x2 on x2.b3 = y1.b3+0
 join t3 as y2 on y2.b3 = x2.c3 and y2.d3 = x1.c3;
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	filtered	Extra
-1	SIMPLE	x1	ALL	b3,c3,c3_2	NULL	NULL	NULL	7	100.00	Parent of 4 pushed join@1
+1	SIMPLE	x1	ALL	b3,c3,c3_2	NULL	NULL	NULL	7	100.00	Parent of 2 pushed join@1
 1	SIMPLE	y1	ref	b3	b3	9	test.x1.b3,test.x1.d3	1	100.00	Child of 'x1' in pushed join@1; Using where
-1	SIMPLE	x2	ref	b3,c3,c3_2	b3	4	test.x1.b3	1	100.00	Child of 'x1' in pushed join@1
-1	SIMPLE	y2	ref	b3	b3	9	test.x2.c3,test.x1.c3	1	100.00	Child of 'x2' in pushed join@1; Using where
+1	SIMPLE	x2	ref	b3,c3,c3_2	b3	4	func	1	100.00	Parent of 2 pushed join@2; Using where
+1	SIMPLE	y2	ref	b3	b3	9	test.x2.c3,test.x1.c3	1	100.00	Child of 'x2' in pushed join@2; Using where
 Warnings:
-Note	1003	select straight_join `test`.`x1`.`a3` AS `a3`,`test`.`x1`.`b3` AS `b3`,`test`.`x1`.`c3` AS `c3`,`test`.`x1`.`d3` AS `d3`,`test`.`y1`.`a3` AS `a3`,`test`.`y1`.`b3` AS `b3`,`test`.`y1`.`c3` AS `c3`,`test`.`y1`.`d3` AS `d3`,`test`.`x2`.`a3` AS `a3`,`test`.`x2`.`b3` AS `b3`,`test`.`x2`.`c3` AS `c3`,`test`.`x2`.`d3` AS `d3`,`test`.`y2`.`a3` AS `a3`,`test`.`y2`.`b3` AS `b3`,`test`.`y2`.`c3` AS `c3`,`test`.`y2`.`d3` AS `d3` from `test`.`t3` `x1` join `test`.`t3` `y1` join `test`.`t3` `x2` join `test`.`t3` `y2` where ((`test`.`y1`.`d3` = `test`.`x1`.`d3`) and (`test`.`y1`.`b3` = `test`.`x1`.`b3`) and (`test`.`x2`.`b3` = `test`.`x1`.`b3`) and (`test`.`y2`.`d3` = `test`.`x1`.`c3`) and (`test`.`y2`.`b3` = `test`.`x2`.`c3`))
+Note	9999	Can't push table 'x2' as child, column 'b3' does neither 'ref' a column nor a constant
+Note	9999	Can't push table 'y2' as child of 'x1', column 'x2.c3' is outside scope of pushable join
+Note	1003	select straight_join `test`.`x1`.`a3` AS `a3`,`test`.`x1`.`b3` AS `b3`,`test`.`x1`.`c3` AS `c3`,`test`.`x1`.`d3` AS `d3`,`test`.`y1`.`a3` AS `a3`,`test`.`y1`.`b3` AS `b3`,`test`.`y1`.`c3` AS `c3`,`test`.`y1`.`d3` AS `d3`,`test`.`x2`.`a3` AS `a3`,`test`.`x2`.`b3` AS `b3`,`test`.`x2`.`c3` AS `c3`,`test`.`x2`.`d3` AS `d3`,`test`.`y2`.`a3` AS `a3`,`test`.`y2`.`b3` AS `b3`,`test`.`y2`.`c3` AS `c3`,`test`.`y2`.`d3` AS `d3` from `test`.`t3` `x1` join `test`.`t3` `y1` join `test`.`t3` `x2` join `test`.`t3` `y2` where ((`test`.`y1`.`d3` = `test`.`x1`.`d3`) and (`test`.`y1`.`b3` = `test`.`x1`.`b3`) and (`test`.`y2`.`d3` = `test`.`x1`.`c3`) and (`test`.`y2`.`b3` = `test`.`x2`.`c3`) and (`test`.`x2`.`b3` = (`test`.`x1`.`b3` + 0)))
 select straight_join * from
 t3 as x1
-join t3 as y1 on x1.b3 = y1.b3 and x1.d3 = y1.d3
-join t3 as x2 on x2.b3 = y1.b3
+join t3 as y1 on y1.b3 = x1.b3 and y1.d3 = x1.d3
+join t3 as x2 on x2.b3 = y1.b3+0
 join t3 as y2 on y2.b3 = x2.c3 and y2.d3 = x1.c3;
 a3	b3	c3	d3	a3	b3	c3	d3	a3	b3	c3	d3	a3	b3	c3	d3
 prepare stmt1 from
@@ -4773,21 +4775,21 @@ PRIMARY KEY (`a`,`b`)
 insert into t2 values (0), (1), (2), (3), (4), (5), (6), (7), (8), (9);
 insert into t3 select 1, x1.a * 10+x2.a from t2 as x1 cross join t2 as x2;
 explain select straight_join count(*) from t1 as x0  
-join t3 as x1 on x0.c=x1.a  
-join t1 as x2 on x0.c=x2.a 
-join t3 as x3 on x2.c=x3.a  
-join t1 as x4 on x0.d=x4.a and x3.b=x4.b;
+join t3 as x1 on x1.a=x0.c
+join t1 as x2 on x2.a=x0.d
+join t3 as x3 on x3.a=x2.c
+join t1 as x4 on x4.a=x0.d and x4.b=x3.b;
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
 1	SIMPLE	x0	ALL	NULL	NULL	NULL	NULL	4	Parent of 5 pushed join@1
 1	SIMPLE	x1	ref	PRIMARY	PRIMARY	4	test.x0.c	1	Child of 'x0' in pushed join@1
-1	SIMPLE	x2	ref	PRIMARY	PRIMARY	4	test.x0.c	1	Child of 'x0' in pushed join@1
+1	SIMPLE	x2	ref	PRIMARY	PRIMARY	4	test.x0.d	1	Child of 'x0' in pushed join@1
 1	SIMPLE	x3	ref	PRIMARY	PRIMARY	4	test.x2.c	1	Child of 'x2' in pushed join@1
 1	SIMPLE	x4	eq_ref	PRIMARY	PRIMARY	8	test.x0.d,test.x3.b	1	Child of 'x3' in pushed join@1
 select straight_join count(*) from t1 as x0  
-join t3 as x1 on x0.c=x1.a  
-join t1 as x2 on x0.c=x2.a 
-join t3 as x3 on x2.c=x3.a  
-join t1 as x4 on x0.d=x4.a and x3.b=x4.b;
+join t3 as x1 on x1.a=x0.c
+join t1 as x2 on x2.a=x0.d
+join t3 as x3 on x3.a=x2.c
+join t1 as x4 on x4.a=x0.d and x4.b=x3.b;
 count(*)
 4800
 drop table t1;
@@ -5471,7 +5473,7 @@ counter_name	spj_counts_at_end.val - spj_counts_at_startup.val
 CONST_PRUNED_RANGE_SCANS_RECEIVED	6
 LOCAL_TABLE_SCANS_SENT	248
 PRUNED_RANGE_SCANS_RECEIVED	25
-RANGE_SCANS_RECEIVED	722
+RANGE_SCANS_RECEIVED	728
 READS_RECEIVED	58
 TABLE_SCANS_RECEIVED	248
 drop table spj_counts_at_startup;
@@ -5483,9 +5485,9 @@ pruned_scan_count
 sorted_scan_count
 10
 pushed_queries_defined
-397
+399
 pushed_queries_dropped
 11
 pushed_queries_executed
-544
+547
 set ndb_join_pushdown = @save_ndb_join_pushdown;

mysql-test/suite/ndb/t/ndb_join_pushdown.test

@@ -1025,10 +1025,10 @@ connection spj;
 
 explain select count(*) from t1 
   join t1 as t2 on t2.a = t1.c 
-  join t1 as t3 on t3.a = t1.c;
+  join t1 as t3 on t3.a = t1.d;
 select count(*) from t1 
   join t1 as t2 on t2.a = t1.c 
-  join t1 as t3 on t3.a = t1.c;
+  join t1 as t3 on t3.a = t1.d;
 
 # Test bushy join with pruned scan and larger result set.
 
@@ -1043,14 +1043,14 @@ delete from t1;
 
 insert into tx values (0), (1), (2), (3), (4), (5), (6), (7), (8), (9);
 
-insert into t1 select 1, x1.a * 10+x2.a, 1, 0 from tx as x1 cross join tx as x2;
+insert into t1 select 1, x1.a * 10+x2.a, 1, 1 from tx as x1 cross join tx as x2;
 
 explain select count(*) from t1 as x1
   join t1 as x2 on x2.a = x1.c and x1.b < 2 
-  join t1 as x3 on x3.a = x1.c;
+  join t1 as x3 on x3.a = x1.d;
 select count(*) from t1 as x1
   join t1 as x2 on x2.a = x1.c and x1.b < 2 
-  join t1 as x3 on x3.a = x1.c;
+  join t1 as x3 on x3.a = x1.d;
 
 connection ddl;
 drop table t1;
@@ -1656,14 +1656,14 @@ select straight_join *
 explain extended
 select straight_join * from
   t3 as x1
-  join t3 as y1 on x1.b3 = y1.b3 and x1.d3 = y1.d3
-  join t3 as x2 on x2.b3 = y1.b3
+  join t3 as y1 on y1.b3 = x1.b3 and y1.d3 = x1.d3
+  join t3 as x2 on x2.b3 = y1.b3+0
   join t3 as y2 on y2.b3 = x2.c3 and y2.d3 = x1.c3;
 --sorted_result
 select straight_join * from
   t3 as x1
-  join t3 as y1 on x1.b3 = y1.b3 and x1.d3 = y1.d3
-  join t3 as x2 on x2.b3 = y1.b3
+  join t3 as y1 on y1.b3 = x1.b3 and y1.d3 = x1.d3
+  join t3 as x2 on x2.b3 = y1.b3+0
   join t3 as y2 on y2.b3 = x2.c3 and y2.d3 = x1.c3;
 
 
@@ -3312,16 +3312,16 @@ connection spj;
 insert into t3 select 1, x1.a * 10+x2.a from t2 as x1 cross join t2 as x2;
 
 explain select straight_join count(*) from t1 as x0  
-   join t3 as x1 on x0.c=x1.a  
-   join t1 as x2 on x0.c=x2.a 
-   join t3 as x3 on x2.c=x3.a  
-   join t1 as x4 on x0.d=x4.a and x3.b=x4.b;
+   join t3 as x1 on x1.a=x0.c
+   join t1 as x2 on x2.a=x0.d
+   join t3 as x3 on x3.a=x2.c
+   join t1 as x4 on x4.a=x0.d and x4.b=x3.b;
 
 select straight_join count(*) from t1 as x0  
-   join t3 as x1 on x0.c=x1.a  
-   join t1 as x2 on x0.c=x2.a 
-   join t3 as x3 on x2.c=x3.a  
-   join t1 as x4 on x0.d=x4.a and x3.b=x4.b;
+   join t3 as x1 on x1.a=x0.c
+   join t1 as x2 on x2.a=x0.d
+   join t3 as x3 on x3.a=x2.c
+   join t1 as x4 on x4.a=x0.d and x4.b=x3.b;
 
 connection ddl;
 drop table t1;

sql/abstract_query_plan.cc

@@ -261,6 +261,33 @@ namespace AQP
     return get_join_tab()->table;
   }
 
+  double Table_access::get_fanout() const
+  {
+    switch (get_access_type())
+    {
+      case AT_PRIMARY_KEY:
+      case AT_UNIQUE_KEY:
+        return 1.0;
+
+      case AT_ORDERED_INDEX_SCAN:
+        DBUG_ASSERT(get_join_tab()->join->best_positions[m_tab_no].records_read>0.0);
+        return get_join_tab()->join->best_positions[m_tab_no].records_read;
+
+      case AT_MULTI_PRIMARY_KEY:
+      case AT_MULTI_UNIQUE_KEY:
+      case AT_MULTI_MIXED:
+        DBUG_ASSERT(get_join_tab()->join->best_positions[m_tab_no].records_read>0.0);
+        return get_join_tab()->join->best_positions[m_tab_no].records_read;
+
+      case AT_TABLE_SCAN:
+        DBUG_ASSERT(get_join_tab()->table->file->stats.records>0.0);
+        return static_cast<double>(get_join_tab()->table->file->stats.records);
+
+      default:
+        return 99999999.0;
+    }
+  }
+
   /** Get the JOIN_TAB object that corresponds to this operation.*/
   const JOIN_TAB* Table_access::get_join_tab() const
   {

sql/abstract_query_plan.h

@@ -205,6 +205,8 @@ namespace AQP
 
     TABLE* get_table() const;
 
+    double get_fanout() const;
+
     Item_equal* get_item_equal(const Item_field* field_item) const;
 
     void dbug_print() const;

sql/ha_ndbcluster_push.cc

@@ -1063,12 +1063,20 @@ ndb_pushed_builder_ctx::optimize_query_plan()
   DBUG_ENTER("optimize_query_plan");
   const uint root_no= m_join_root->get_access_no();
 
+  for (uint tab_no= root_no; tab_no<m_plan.get_access_count(); tab_no++)
+  {
+    if (m_join_scope.contain(tab_no))
+    {
+      m_tables[tab_no].m_fanout = m_plan.get_table_access(tab_no)->get_fanout();
+      m_tables[tab_no].m_child_fanout = 1.0;
+    }
+  }
+
   // Find an optimal order for joining the tables
   for (uint tab_no= m_plan.get_access_count()-1;
        tab_no > root_no;
        tab_no--)
   {
-    struct pushed_tables &table= m_tables[tab_no];
     if (!m_join_scope.contain(tab_no))
       continue;
 
@@ -1078,6 +1086,7 @@ ndb_pushed_builder_ctx::optimize_query_plan()
      * don't skip any dependent parents from our ancestors
      * when selecting the actuall 'm_parent' to be used.
      */
+    pushed_tables &table= m_tables[tab_no];
     if (!table.m_depend_parents.is_clear_all())
     {
       ndb_table_access_map const &dependency= table.m_depend_parents;
@@ -1115,12 +1124,40 @@ ndb_pushed_builder_ctx::optimize_query_plan()
 
     /**
      * In order to take advantage of the parallelism in the SPJ block; 
-     * Choose the first possible parent candidate. Will result in the
-     * most 'bushy' query plan (aka: star-join)
+     * Initial parent candidate is the first possible among 'parents'.
+     * Will result in the most 'bushy' query plan (aka: star-join)
      */
     parent_no= parents.first_table(root_no);
+
+    if (table.m_fanout*table.m_child_fanout > 1.0 ||
+        !ndbcluster_is_lookup_operation(m_plan.get_table_access(tab_no)->get_access_type()))
+    {
+      /**
+       * This is a index-scan or lookup with scan childs.
+       * Push optimization for index-scan execute:
+       *
+       * These are relative expensive operation which we try to avoid to 
+       * execute whenever possible. By making them depending on parent 
+       * operations with high selectivity, they will be eliminated when
+       * the parent returns no matching rows.
+       *
+       * -> Execute index-scan after any such parents
+       */
+      for (uint candidate= parent_no+1; candidate<parents.length(); candidate++)
+      {
+        if (parents.contain(candidate))
+        {
+          if (m_tables[candidate].m_fanout > 1.0)
+            break;
+
+          parent_no= candidate;     // Parent candidate is selective, eval after
+        }
+      }
+    }
+
     DBUG_ASSERT(parent_no < tab_no);
     table.m_parent= parent_no;
+    m_tables[parent_no].m_child_fanout*= table.m_fanout*table.m_child_fanout;
 
     ndb_table_access_map dependency(table.m_depend_parents);
     dependency.clear_bit(parent_no);
@@ -1134,7 +1171,7 @@ ndb_pushed_builder_ctx::optimize_query_plan()
   {
     if (m_join_scope.contain(tab_no))
     {
-      struct pushed_tables &table= m_tables[tab_no];
+      pushed_tables &table= m_tables[tab_no];
       const uint parent_no= table.m_parent;
       table.m_ancestors= m_tables[parent_no].m_ancestors;
       table.m_ancestors.add(parent_no);

sql/ha_ndbcluster_push.h

@@ -283,6 +283,8 @@ class ndb_pushed_builder_ctx
       m_depend_parents(), 
       m_parent(MAX_TABLES), 
       m_ancestors(), 
+      m_fanout(1.0),
+      m_child_fanout(1.0),
       m_op(NULL) 
     {}
 
@@ -317,6 +319,16 @@ class ndb_pushed_builder_ctx
      */
     ndb_table_access_map m_ancestors;
 
+    /**
+     * The fanout of this table.
+     */
+    double m_fanout;
+
+    /**
+     * The (cross) product of all child fanouts.
+     */
+    double m_child_fanout;
+
     const NdbQueryOperationDef* m_op;
   } m_tables[MAX_TABLES];