Bug#11761614 MYSQLD SEGFAULTS WHEN BUILT USING

  --WITH-MAX-INDEXES=128

Post-push fix for test failure on Solaris.

Author: Nirbhay Choubey

mysql-test/CMakeLists.txt

@@ -128,8 +128,10 @@ ENDFOREACH()
 # NOTE: While committing a patch please make sure that the file is unmodified
 # and should show the default MAX_INDEXES (i.e. 64U).
 IF (MAX_INDEXES)
-  FILE(WRITE include/max_indexes.inc
-       "# Warning: This is an auto-generated file. Please do not modify it.
+  IF(NOT (${MAX_INDEXES} EQUAL 64U))
+    FILE(WRITE include/max_indexes.inc
+         "# Warning: This is an auto-generated file. Please do not modify it.
 --let $max_indexes = ${MAX_INDEXES}\n")
-  MESSAGE(STATUS "mysql-test/include/max_indexes.inc adjusted")
+    MESSAGE(STATUS "mysql-test/include/max_indexes.inc adjusted")
+  ENDIF()
 ENDIF()

sql/sql_bitmap.h

@@ -58,7 +58,17 @@ template <uint default_width> class Bitmap
     bitmap_init(&map2, (uint32 *) &buf2, sizeof(ulonglong) * 8, 0);
 
     // Store the original bits.
-    buf2= map2buff;
+    if (sizeof(ulonglong) >= 8) {
+      int8store(const_cast<uchar *>(static_cast<uchar *>
+                                    (static_cast<void *>(&buf2))),
+                map2buff);
+    } else {
+      DBUG_ASSERT(sizeof(buffer) >= 4);
+      int4store(const_cast<uchar *>(static_cast<uchar *>
+                                    (static_cast<void *>(&buf2))),
+                map2buff);
+    }
+
     bitmap_intersect(&map, &map2);
   }
   /* Use highest bit for all bits above sizeof(ulonglong)*8. */
@@ -99,9 +109,12 @@ template <uint default_width> class Bitmap
   ulonglong to_ulonglong() const
   {
     if (sizeof(buffer) >= 8)
-      return uint8korr(buffer);
+      return uint8korr(static_cast<const uchar *>
+                       (static_cast<const void *>(buffer)));
     DBUG_ASSERT(sizeof(buffer) >= 4);
-    return (ulonglong) uint4korr(buffer);
+    return (ulonglong)
+      uint4korr(static_cast<const uchar *>
+                (static_cast<const void *>(buffer)));
   }
   uint bits_set() const { return bitmap_bits_set(&map); }
 };

unittest/gunit/bitmap-t.cc

@@ -42,7 +42,7 @@ class BitmapTest : public ::testing::Test
 TEST_F(BitmapTest, IntersectTest)
 {
   bitmap.set_prefix(4);
-  bitmap.intersect(0xBBBB);
+  bitmap.intersect(0xBBBBULL);
   EXPECT_TRUE(bitmap.is_set(0));
   EXPECT_TRUE(bitmap.is_set(1));
   EXPECT_FALSE(bitmap.is_set(2));
@@ -53,19 +53,19 @@ TEST_F(BitmapTest, IntersectTest)
   EXPECT_TRUE(bitmap.is_clear_all());
 }
 
-TEST_F(BitmapTest, UULTest)
+TEST_F(BitmapTest, ULLTest)
 {
   bitmap.set_all();
-  bitmap.intersect(0x0123456789ABCDEF);
-  ulonglong uul= bitmap.to_ulonglong();
-  EXPECT_TRUE(uul == 0x0123456789ABCDEF);
+  bitmap.intersect(0x0123456789ABCDEFULL);
+  ulonglong ull= bitmap.to_ulonglong();
+  EXPECT_TRUE(ull == 0x0123456789ABCDEFULL);
 
   Bitmap<24> bitmap24;
   bitmap24.init();
   bitmap24.set_all();
-  bitmap24.intersect(0x47B);
-  ulonglong uul24= bitmap24.to_ulonglong();
-  EXPECT_TRUE(uul24 == 0x47B);
+  bitmap24.intersect(0x47BULL);
+  ulonglong ull24= bitmap24.to_ulonglong();
+  EXPECT_TRUE(ull24 == 0x47BULL);
 }
 
 }  // namespace