This is a fix for deviations in statistics counter values that occur when running ndb_join_pushdown.test on

Jan Wedvik · Jan Wedvik · commit b4975b249ace · 2011-09-09T09:23:13.000+02:00
big-endian (Sparc) and with Valgrind.

This fix does two things:

* Some ndb$info counters are removed from the test, because their values depend on endian-dependent key hashing. 
This includes READS_NOT_FOUND, LOCAL_READS_SENT,  REMOTE_READS_SENT and SCAN_ROWS_RETURNED.

* Ndb_scan_count was sometimes off by one when testing with Valgrind, possibly because table statistics was refreshed 
at unpredictable intervals. Instead of testing the exact value of Ndb_scan_count, the test will fail if 
Ndb_scan_count deviates from the expected values by more than 10%.
diff --git a/mysql-test/suite/ndb/r/ndb_join_pushdown.result b/mysql-test/suite/ndb/r/ndb_join_pushdown.result
@@ -4,8 +4,6 @@ select counter_name, sum(val) as val
 from ndbinfo.counters 
 where block_name='DBSPJ' 
 group by counter_name;
-create temporary table spj_save_counts like spj_counts_at_startup;
-insert into spj_save_counts values ('SCAN_ROWS_RETURNED', 0);
 set @save_ndb_join_pushdown = @@session.ndb_join_pushdown;
 set ndb_join_pushdown = true;
 create table t1 (
@@ -2060,7 +2058,6 @@ count(*)
 304
 drop table tx;
 alter table t1 partition by key(a);
-update spj_save_counts set val = (select sum(val) from ndbinfo.counters where block_name='DBSPJ' and counter_name='SCAN_ROWS_RETURNED') where counter_name='SCAN_ROWS_RETURNED';
 explain select count(*) from t1 
 join t1 as t2 on t2.a = t1.c 
 join t1 as t3 on t3.a = t1.c;
@@ -2092,7 +2089,6 @@ join t1 as x2 on x2.a = x1.c and x1.b < 2
 join t1 as x3 on x3.a = x1.c;
 count(*)
 20000
-update spj_counts_at_startup set val = val + (select sum(val) from ndbinfo.counters where block_name='DBSPJ' and counter_name='SCAN_ROWS_RETURNED') - (select val from spj_save_counts where counter_name='SCAN_ROWS_RETURNED') where counter_name='SCAN_ROWS_RETURNED';
 drop table t1;
 drop table tx;
 create table t1 (a int, b int, primary key(a) using hash) engine = ndb;
@@ -4331,7 +4327,6 @@ id	select_type	table	type	possible_keys	key	key_len	ref	rows	filtered	Extra
 Warnings:
 Note	9999	Can't push table 'x3' as child of 'x1', outer join of scan-child not implemented
 Note	1003	select straight_join `test`.`x1`.`pk` AS `pk`,`test`.`x1`.`u` AS `u`,`test`.`x1`.`a` AS `a`,`test`.`x1`.`b` AS `b`,`test`.`x2`.`pk` AS `pk`,`test`.`x2`.`u` AS `u`,`test`.`x2`.`a` AS `a`,`test`.`x2`.`b` AS `b`,`test`.`x3`.`pk` AS `pk`,`test`.`x3`.`u` AS `u`,`test`.`x3`.`a` AS `a`,`test`.`x3`.`b` AS `b` from `test`.`t1` `x1` left join (`test`.`t1` `x2` join `test`.`t1` `x3`) on(((`test`.`x2`.`pk` = `test`.`x1`.`a`) and (`test`.`x3`.`b` = `test`.`x2`.`a`))) where 1
-update spj_save_counts set val = (select sum(val) from ndbinfo.counters where block_name='DBSPJ' and counter_name='SCAN_ROWS_RETURNED') where counter_name='SCAN_ROWS_RETURNED';
 explain extended select straight_join count(*) from t1 as x1 
 join t1 as x2 on x2.b = x1.a
 join t1 as x3 on x3.b = x1.b;
@@ -4381,7 +4376,6 @@ join t1 as x7 on x7.b = x1.a
 where x3.a < x2.pk and x4.a < x3.pk;
 count(*)
 632736
-update spj_counts_at_startup set val = val + (select sum(val) from ndbinfo.counters where block_name='DBSPJ' and counter_name='SCAN_ROWS_RETURNED') - (select val from spj_save_counts where counter_name='SCAN_ROWS_RETURNED') where counter_name='SCAN_ROWS_RETURNED';
 explain extended select straight_join count(*) from t1 as x1 
 left join t1 as x2 on x2.b = x1.a
 join t1 as x3 on x3.b = x1.b;
@@ -4426,7 +4420,6 @@ join t1 as x2 on x2.b = x1.a
 left join t1 as x3 on x3.b = x1.b;
 count(*)
 2028
-update spj_save_counts set val = (select sum(val) from ndbinfo.counters where block_name='DBSPJ' and counter_name='SCAN_ROWS_RETURNED') where counter_name='SCAN_ROWS_RETURNED';
 explain extended
 select straight_join count(*) from t1 as x1
 join t1 as x2 on x2.b = x1.a
@@ -4450,7 +4443,6 @@ join t1 as x2 on x2.b = x1.a
 join t1 as x3 on x3.pk = x1.a join t1 as x4 on x4.b = x3.a;
 count(*)
 2028
-update spj_counts_at_startup set val = val + (select sum(val) from ndbinfo.counters where block_name='DBSPJ' and counter_name='SCAN_ROWS_RETURNED') - (select val from spj_save_counts where counter_name='SCAN_ROWS_RETURNED') where counter_name='SCAN_ROWS_RETURNED';
 explain extended select straight_join count(*) from t1 as x1 
 left join t1 as x3 on x3.b = x1.a
 join t1 as x2 on x2.pk = x1.a;
@@ -4556,15 +4548,13 @@ id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
 1	SIMPLE	x2	ref	PRIMARY	PRIMARY	4	test.x0.c	1	Child of 'x0' in pushed join@1
 1	SIMPLE	x3	ref	PRIMARY	PRIMARY	4	test.x2.c	1	Child of 'x2' in pushed join@1
 1	SIMPLE	x4	eq_ref	PRIMARY	PRIMARY	8	test.x0.d,test.x3.b	1	Child of 'x3' in pushed join@1
-update spj_save_counts set val = (select sum(val) from ndbinfo.counters where block_name='DBSPJ' and counter_name='SCAN_ROWS_RETURNED') where counter_name='SCAN_ROWS_RETURNED';
 select straight_join count(*) from t1 as x0  
 join t3 as x1 on x0.c=x1.a  
 join t1 as x2 on x0.c=x2.a 
 join t3 as x3 on x2.c=x3.a  
 join t1 as x4 on x0.d=x4.a and x3.b=x4.b;
 count(*)
 4800
-update spj_counts_at_startup set val = val + (select sum(val) from ndbinfo.counters where block_name='DBSPJ' and counter_name='SCAN_ROWS_RETURNED') - (select val from spj_save_counts where counter_name='SCAN_ROWS_RETURNED') where counter_name='SCAN_ROWS_RETURNED';
 drop table t1;
 drop table t2;
 drop table t3;
@@ -5235,32 +5225,24 @@ group by counter_name;
 select spj_counts_at_end.counter_name, spj_counts_at_end.val - spj_counts_at_startup.val 
 from spj_counts_at_end, spj_counts_at_startup 
 where spj_counts_at_end.counter_name = spj_counts_at_startup.counter_name
-and spj_counts_at_end.counter_name <> 'LOCAL_READS_SENT'
+and spj_counts_at_end.counter_name <> 'READS_NOT_FOUND'
+       and spj_counts_at_end.counter_name <> 'LOCAL_READS_SENT'
        and spj_counts_at_end.counter_name <> 'REMOTE_READS_SENT'
        and spj_counts_at_end.counter_name <> 'LOCAL_RANGE_SCANS_SENT'
        and spj_counts_at_end.counter_name <> 'REMOTE_RANGE_SCANS_SENT'
+       and spj_counts_at_end.counter_name <> 'SCAN_ROWS_RETURNED'
        and spj_counts_at_end.counter_name <> 'SCAN_BATCHES_RETURNED';
 counter_name	spj_counts_at_end.val - spj_counts_at_startup.val
 CONST_PRUNED_RANGE_SCANS_RECEIVED	6
 LOCAL_TABLE_SCANS_SENT	244
 PRUNED_RANGE_SCANS_RECEIVED	25
 RANGE_SCANS_RECEIVED	720
-READS_NOT_FOUND	6616
 READS_RECEIVED	52
-SCAN_ROWS_RETURNED	76386
 TABLE_SCANS_RECEIVED	244
-select sum(spj_counts_at_end.val - spj_counts_at_startup.val) as 'LOCAL+REMOTE READS_SENT'
-       from spj_counts_at_end, spj_counts_at_startup 
-where spj_counts_at_end.counter_name = spj_counts_at_startup.counter_name
-and (spj_counts_at_end.counter_name = 'LOCAL_READS_SENT'
-       or spj_counts_at_end.counter_name = 'REMOTE_READS_SENT');
-LOCAL+REMOTE READS_SENT
-35288
-drop table spj_save_counts;
 drop table spj_counts_at_startup;
 drop table spj_counts_at_end;
-scan_count
-2524
+scan_count_derived
+1.0
 pruned_scan_count
 8
 sorted_scan_count
diff --git a/mysql-test/suite/ndb/t/ndb_join_pushdown.test b/mysql-test/suite/ndb/t/ndb_join_pushdown.test
@@ -37,30 +37,6 @@ let $pushed_queries_dropped_at_startup = query_get_value(show status like 'Ndb_p
 let $pushed_queries_executed_at_startup = query_get_value(show status like 'Ndb_pushed_queries_executed', Value, 1);
 let $pushed_reads_at_startup = query_get_value(show status like 'Ndb_pushed_reads', Value, 1);
 
-# Use this table and the two queries below to turn of meassuring 
-# SCAN_ROWS_RETURNED for certain bushy scan queries. The reason for this is 
-# that growth of this counter is platform dependent for these queries. There
-# are two reasons for this:
-# 1. Distribution hashing (partitioning) of tables is endian dependent. This
-# may cause data to be more skewed on some platforms. This again requires more
-# batches to scan the table and thus more repeats of repeatable scans (i.e. 
-# those that will be repeated for each batch of the other branch of a bushy
-# scan). This increases the overall scan row count.
-# 2. If a timer expires in LQH after receiving SCAN_FRAGREQ, LQH may decide to
-# send SCAN_FRAGCONF immediately, even if more tuples could fit in the batch.
-# As above this causes more repeats of repeatable scans.
-
-create temporary table spj_save_counts like spj_counts_at_startup;
-
-insert into spj_save_counts values ('SCAN_ROWS_RETURNED', 0);
-
-# Record current counter value.
-let $save_scan_rows_returned = update spj_save_counts set val = (select sum(val) from ndbinfo.counters where block_name='DBSPJ' and counter_name='SCAN_ROWS_RETURNED') where counter_name='SCAN_ROWS_RETURNED';
-
-# Update spj_counts_at_startup to compensate for counter increments since 
-# running save_scan_rows_returned.
-let $compensate_scan_rows_returned = update spj_counts_at_startup set val = val + (select sum(val) from ndbinfo.counters where block_name='DBSPJ' and counter_name='SCAN_ROWS_RETURNED') - (select val from spj_save_counts where counter_name='SCAN_ROWS_RETURNED') where counter_name='SCAN_ROWS_RETURNED';
-
 ##############
 # Test start
 
@@ -1046,7 +1022,6 @@ connection ddl;
 alter table t1 partition by key(a);
 
 connection spj;
-eval $save_scan_rows_returned;
 
 explain select count(*) from t1 
   join t1 as t2 on t2.a = t1.c 
@@ -1077,8 +1052,6 @@ select count(*) from t1 as x1
   join t1 as x2 on x2.a = x1.c and x1.b < 2 
   join t1 as x3 on x3.a = x1.c;
 
-eval $compensate_scan_rows_returned;
-
 connection ddl;
 drop table t1;
 drop table tx;
@@ -2987,7 +2960,6 @@ explain extended select straight_join * from
 # These should be allowed to be executed in 'parallel', depending on
 # only the root operation
 #
-eval $save_scan_rows_returned;
 
 explain extended select straight_join count(*) from t1 as x1 
   join t1 as x2 on x2.b = x1.a
@@ -3023,8 +2995,6 @@ select straight_join count(*) from t1 as x1
   join t1 as x7 on x7.b = x1.a
 where x3.a < x2.pk and x4.a < x3.pk; 
 
-eval $compensate_scan_rows_returned;
-
 #############
 # If we have an outer join, we can't create an artificial dep. 'through' the outer join.
 # In this case the child scan can't be part of the pushed query.
@@ -3064,8 +3034,6 @@ select straight_join count(*) from t1 as x1
 #
 # Bushy execution is expected for these scans (x2 & x4) wrt. root (x1)
 #
-eval $save_scan_rows_returned;
-
 explain extended
 select straight_join count(*) from t1 as x1
   join t1 as x2 on x2.b = x1.a
@@ -3080,8 +3048,6 @@ select straight_join count(*) from t1 as x1
   join t1 as x2 on x2.b = x1.a
   join t1 as x3 on x3.pk = x1.a join t1 as x4 on x4.b = x3.a;
 
-eval $compensate_scan_rows_returned;
-
 #############
 # Test bushy lookups + 1scan, 
 # (Regression test for previous commit: http://lists.mysql.com/commits/117571)
@@ -3195,16 +3161,12 @@ explain select straight_join count(*) from t1 as x0
    join t3 as x3 on x2.c=x3.a  
    join t1 as x4 on x0.d=x4.a and x3.b=x4.b;
 
-eval $save_scan_rows_returned;
-
 select straight_join count(*) from t1 as x0  
    join t3 as x1 on x0.c=x1.a  
    join t1 as x2 on x0.c=x2.a 
    join t3 as x3 on x2.c=x3.a  
    join t1 as x4 on x0.d=x4.a and x3.b=x4.b;
 
-eval $compensate_scan_rows_returned;
-
 connection ddl;
 drop table t1;
 drop table t2;
@@ -3720,21 +3682,15 @@ connection spj;
 select spj_counts_at_end.counter_name, spj_counts_at_end.val - spj_counts_at_startup.val 
        from spj_counts_at_end, spj_counts_at_startup 
        where spj_counts_at_end.counter_name = spj_counts_at_startup.counter_name
+       and spj_counts_at_end.counter_name <> 'READS_NOT_FOUND'
        and spj_counts_at_end.counter_name <> 'LOCAL_READS_SENT'
        and spj_counts_at_end.counter_name <> 'REMOTE_READS_SENT'
        and spj_counts_at_end.counter_name <> 'LOCAL_RANGE_SCANS_SENT'
        and spj_counts_at_end.counter_name <> 'REMOTE_RANGE_SCANS_SENT'
+       and spj_counts_at_end.counter_name <> 'SCAN_ROWS_RETURNED'
        and spj_counts_at_end.counter_name <> 'SCAN_BATCHES_RETURNED';
 
-# The sum of LOCAL+REMOTE should be constant
-select sum(spj_counts_at_end.val - spj_counts_at_startup.val) as 'LOCAL+REMOTE READS_SENT'
-       from spj_counts_at_end, spj_counts_at_startup 
-       where spj_counts_at_end.counter_name = spj_counts_at_startup.counter_name
-       and (spj_counts_at_end.counter_name = 'LOCAL_READS_SENT'
-       or spj_counts_at_end.counter_name = 'REMOTE_READS_SENT');
-
 connection spj;
-drop table spj_save_counts;
 drop table spj_counts_at_startup;
 drop table spj_counts_at_end;
 
@@ -3750,7 +3706,10 @@ let $pushed_reads_at_end = query_get_value(show status like 'Ndb_pushed_reads',
 
 # Calculate the change.
 --disable_query_log
---eval select $scan_count_at_end - $scan_count_at_startup as scan_count
+# There is some random variation in scan_count, probably due to statistics
+# being updated at unpredictable intervals. Therefore, we only test for 
+# deviations greater than on tenth of the expected value.
+--eval select round(($scan_count_at_end - $scan_count_at_startup)/2524, 1) as scan_count_derived
 --eval select $pruned_scan_count_at_end - $pruned_scan_count_at_startup as pruned_scan_count
 --eval select $sorted_scan_count_at_end - $sorted_scan_count_at_startup as sorted_scan_count
 --eval select $pushed_queries_defined_at_end - $pushed_queries_defined_at_startup as pushed_queries_defined
