Bug #32507117 INDEX SIZE LARGER THAN 767 BYTES ALLOWED FOR INNODB WITH REDUNDANT ROW_FORMAT

PROBLEM
-------

1. When doing an alter on a table which was built with one specific
   row type the alter checks the validity of size on the index with
   the row format defined by the global variable innodb_default_row_format
   which is different than the one with which table was built on.

FIX
---

1. Check with the correct format of row type with which the table
   was built in case of inplace
2. In case of copy chose the row format not specified
   by the user choose the format defined by
   innodb_default_row_format

#rb 26179

Reviewed by : Kevin Lewis <kevin.lewis@oracle.com>

Author: Aditya A

mysql-test/suite/innodb/r/alter_row_format.result

@@ -0,0 +1,39 @@
+#
+# Bug #32507117	INDEX SIZE LARGER THAN 767 BYTES ALLOWED FOR INNODB WITH REDUNDANT ROW_FORMAT
+#
+SET @orig_default_row_format = @@global.innodb_default_row_format;
+set global innodb_default_row_format='redundant';
+CREATE TABLE `t1` (
+`id` int unsigned NOT NULL AUTO_INCREMENT,
+`comment` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT
+NULL DEFAULT '',
+PRIMARY KEY (`id`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8;
+CREATE TABLE `t2` (
+`id` int unsigned NOT NULL AUTO_INCREMENT,
+`comment` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT
+NULL DEFAULT '',
+PRIMARY KEY (`id`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8;
+insert into t1(comment) values('k');
+insert into t2(comment) values('k');
+create index idx123 on t1 (`comment`);
+ERROR HY000: Index column size too large. The maximum column size is 767 bytes.
+set global innodb_default_row_format='dynamic';
+create index idx123 on t1 (`comment`);
+ERROR HY000: Index column size too large. The maximum column size is 767 bytes.
+alter table t1 add index idx123 (`comment`);
+ERROR HY000: Index column size too large. The maximum column size is 767 bytes.
+alter table t1 add index idx123 (`comment`), algorithm=copy ;
+SELECT ROW_FORMAT FROM INFORMATION_SCHEMA.INNODB_SYS_TABLES WHERE NAME LIKE 'test/t1';
+ROW_FORMAT
+Dynamic
+SELECT ROW_FORMAT FROM INFORMATION_SCHEMA.INNODB_SYS_TABLES WHERE NAME LIKE 'test/t2';
+ROW_FORMAT
+Redundant
+alter table t2 ROW_FORMAT=dynamic;
+SELECT ROW_FORMAT FROM INFORMATION_SCHEMA.INNODB_SYS_TABLES WHERE NAME LIKE 'test/t2';
+ROW_FORMAT
+Dynamic
+drop table t1,t2;
+SET @@global.innodb_default_row_format=@orig_default_row_format;

mysql-test/suite/innodb/t/alter_row_format.test

@@ -0,0 +1,44 @@
+--echo #
+--echo # Bug #32507117	INDEX SIZE LARGER THAN 767 BYTES ALLOWED FOR INNODB WITH REDUNDANT ROW_FORMAT
+--echo #
+SET @orig_default_row_format = @@global.innodb_default_row_format;
+set global innodb_default_row_format='redundant';
+
+CREATE TABLE `t1` (
+  `id` int unsigned NOT NULL AUTO_INCREMENT,
+  `comment` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT
+NULL DEFAULT '',
+  PRIMARY KEY (`id`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8;
+
+
+CREATE TABLE `t2` (
+  `id` int unsigned NOT NULL AUTO_INCREMENT,
+  `comment` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT
+NULL DEFAULT '',
+  PRIMARY KEY (`id`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8;
+
+insert into t1(comment) values('k');
+insert into t2(comment) values('k');
+
+--error ER_INDEX_COLUMN_TOO_LONG
+create index idx123 on t1 (`comment`);
+
+set global innodb_default_row_format='dynamic';
+--error ER_INDEX_COLUMN_TOO_LONG
+create index idx123 on t1 (`comment`);
+
+--error ER_INDEX_COLUMN_TOO_LONG
+alter table t1 add index idx123 (`comment`);
+
+alter table t1 add index idx123 (`comment`), algorithm=copy ;
+
+SELECT ROW_FORMAT FROM INFORMATION_SCHEMA.INNODB_SYS_TABLES WHERE NAME LIKE 'test/t1';
+
+SELECT ROW_FORMAT FROM INFORMATION_SCHEMA.INNODB_SYS_TABLES WHERE NAME LIKE 'test/t2';
+alter table t2 ROW_FORMAT=dynamic;
+SELECT ROW_FORMAT FROM INFORMATION_SCHEMA.INNODB_SYS_TABLES WHERE NAME LIKE 'test/t2';
+
+drop table t1,t2;
+SET @@global.innodb_default_row_format=@orig_default_row_format;

storage/innobase/handler/handler0alter.cc

@@ -5691,7 +5691,17 @@ ha_innobase::prepare_inplace_alter_table(
 		goto err_exit_no_heap;
 	}
 
-	max_col_len = DICT_MAX_FIELD_LEN_BY_FORMAT_FLAG(info.flags());
+
+	/* The copy algorithm uses the default row format while
+	in-place uses the current format. Find the limit for
+	the resulting format.*/
+	if (innobase_need_rebuild(ha_alter_info)) {
+		max_col_len =
+			DICT_MAX_FIELD_LEN_BY_FORMAT_FLAG(info.flags());
+	} else {
+		max_col_len =
+			DICT_MAX_FIELD_LEN_BY_FORMAT_FLAG(m_prebuilt->table->flags);
+	}
 
 	/* Check each index's column length to make sure they do not
 	exceed limit */