MErging from mysql-5.5-security

Author: Alexander Barkov

mysql-test/r/ctype_ujis.result

@@ -2374,6 +2374,16 @@ hex(convert(_latin1 0xA4A2 using ujis))	hex(c2)
 DROP PROCEDURE sp1;
 DROP TABLE t1;
 DROP TABLE t2;
+#
+# Bug#57257 Replace(ExtractValue(...)) causes MySQL crash
+#
+SET NAMES utf8;
+SELECT CONVERT(REPLACE(EXPORT_SET('a','a','a','','a'),'00','') USING ujis);
+CONVERT(REPLACE(EXPORT_SET('a','a','a','','a'),'00','') USING ujis)
+
+Warnings:
+Warning	1292	Truncated incorrect INTEGER value: 'a'
+Warning	1292	Truncated incorrect INTEGER value: 'a'
 set names default;
 set character_set_database=default;
 set character_set_server=default;

mysql-test/r/ctype_utf8.result

@@ -4976,6 +4976,17 @@ maketime(`a`,`a`,`a`)
 DROP TABLE t1;
 SET sql_mode=default;
 #
+# Bug#57687 crash when reporting duplicate group_key error and utf8
+# Make sure to modify this when Bug#58081 is fixed.
+#
+SET NAMES utf8;
+CREATE TABLE t1 (a INT);
+INSERT INTO t1 VALUES (0), (0), (1), (0), (0);
+SELECT COUNT(*) FROM t1, t1 t2 
+GROUP BY INSERT('', t2.a, t1.a, (@@global.max_binlog_size));
+ERROR 23000: Duplicate entry '107374182410737418241' for key 'group_key'
+DROP TABLE t1;
+#
 # End of 5.5 tests
 #
 #

mysql-test/r/xml.result

@@ -1093,4 +1093,17 @@ Warnings:
 Warning	1525	Incorrect XML value: 'parse error at line 1 pos 23: unexpected END-OF-INPUT'
 Warning	1525	Incorrect XML value: 'parse error at line 1 pos 23: unexpected END-OF-INPUT'
 DROP TABLE t1;
+#
+# Bug#57257 Replace(ExtractValue(...)) causes MySQL crash
+#
+SET NAMES utf8;
+SELECT REPLACE(EXTRACTVALUE('1', '/a'),'ds','');
+REPLACE(EXTRACTVALUE('1', '/a'),'ds','')
+
+#
+# Bug #57820 extractvalue crashes
+#
+SELECT AVG(DISTINCT EXTRACTVALUE((''),('$@k')));
+AVG(DISTINCT EXTRACTVALUE((''),('$@k')))
+NULL
 End of 5.1 tests

mysql-test/t/ctype_ujis.test

@@ -1209,6 +1209,13 @@ DROP PROCEDURE sp1;
 DROP TABLE t1;
 DROP TABLE t2;
 
+--echo #
+--echo # Bug#57257 Replace(ExtractValue(...)) causes MySQL crash
+--echo #
+SET NAMES utf8;
+SELECT CONVERT(REPLACE(EXPORT_SET('a','a','a','','a'),'00','') USING ujis);
+
+
 set names default;
 set character_set_database=default;
 set character_set_server=default;

mysql-test/t/ctype_utf8.test

@@ -1549,6 +1549,18 @@ DROP TABLE t1, t2;
 SET NAMES utf8;
 --source include/ctype_numconv.inc
 
+--echo #
+--echo # Bug#57687 crash when reporting duplicate group_key error and utf8
+--echo # Make sure to modify this when Bug#58081 is fixed.
+--echo #
+SET NAMES utf8;
+CREATE TABLE t1 (a INT);
+INSERT INTO t1 VALUES (0), (0), (1), (0), (0);
+--error ER_DUP_ENTRY
+SELECT COUNT(*) FROM t1, t1 t2 
+GROUP BY INSERT('', t2.a, t1.a, (@@global.max_binlog_size));
+DROP TABLE t1;
+
 
 --echo #
 --echo # End of 5.5 tests

mysql-test/t/xml.test

@@ -617,4 +617,15 @@ FROM t1 ORDER BY t1.id;
 
 DROP TABLE t1;
 
+--echo #
+--echo # Bug#57257 Replace(ExtractValue(...)) causes MySQL crash
+--echo #
+SET NAMES utf8;
+SELECT REPLACE(EXTRACTVALUE('1', '/a'),'ds','');
+
+--echo #
+--echo # Bug #57820 extractvalue crashes
+--echo #
+SELECT AVG(DISTINCT EXTRACTVALUE((''),('$@k')));
+
 --echo End of 5.1 tests

sql/item_strfunc.cc

@@ -1080,9 +1080,15 @@ String *Item_func_replace::val_str(String *str)
     search=res2->ptr();
     search_end=search+from_length;
 redo:
+    DBUG_ASSERT(res->ptr() || !offset);
     ptr=res->ptr()+offset;
     strend=res->ptr()+res->length();
-    end=strend-from_length+1;
+    /*
+      In some cases val_str() can return empty string
+      with ptr() == NULL and length() == 0.
+      Let's check strend to avoid overflow.
+    */
+    end= strend ? strend - from_length + 1 : NULL;
     while (ptr < end)
     {
         if (*ptr == *search)

sql/item_xmlfunc.cc

@@ -2798,12 +2798,12 @@ String *Item_func_xml_extractvalue::val_str(String *str)
   null_value= 0;
   if (!nodeset_func ||
       !(res= args[0]->val_str(str)) || 
-      !parse_xml(res, &pxml))
+      !parse_xml(res, &pxml) ||
+      !(res= nodeset_func->val_str(&tmp_value)))
   {
     null_value= 1;
     return 0;
   }
-  res= nodeset_func->val_str(&tmp_value);
   return res;  
 }
 

sql/key.cc

@@ -364,9 +364,7 @@ void key_unpack(String *to,TABLE *table,uint idx)
         while (tmp_end > tmp.ptr() && !*--tmp_end) ;
         tmp.length(tmp_end - tmp.ptr() + 1);
       }
-      if (cs->mbmaxlen > 1 &&
-          table->field[key_part->fieldnr - 1]->field_length !=
-          key_part->length)
+      if (cs->mbmaxlen > 1 && (key_part->key_part_flag & HA_PART_KEY_SEG))
       {
         /* 
           Prefix key, multi-byte charset.