diff --git a/ext/mbstring/mbstring.c b/ext/mbstring/mbstring.c index 7172656d736d8..f26ecbbcacb81 100644 --- a/ext/mbstring/mbstring.c +++ b/ext/mbstring/mbstring.c @@ -4586,6 +4586,504 @@ MBSTRING_API bool php_mb_check_encoding(const char *input, size_t length, const /* If we are building an AVX2-only binary, don't compile the next function */ #ifndef ZEND_INTRIN_AVX2_NATIVE +bool utf8_naive(const unsigned char *p, size_t length) +{ + /* This UTF-8 validation function is derived from PCRE2 */ + /* Table of the number of extra bytes, indexed by the first byte masked with + 0x3f. The highest number for a valid UTF-8 first byte is in fact 0x3d. */ + static const uint8_t utf8_table[] = { + 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, + 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, + 2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2, + 3,3,3,3,3,3,3,3 + }; + + for (; length > 0; p++) { + uint32_t d; + unsigned char c = *p; + length--; + + if (c < 128) { + /* ASCII character */ + continue; + } + + if (c < 0xc0) { + /* Isolated 10xx xxxx byte */ + return false; + } + + if (c >= 0xf5) { + return false; + } + + uint32_t ab = utf8_table[c & 0x3f]; /* Number of additional bytes (1-3) */ + if (length < ab) { + /* Missing bytes */ + return false; + } + length -= ab; + + /* Check top bits in the second byte */ + if (((d = *(++p)) & 0xc0) != 0x80) { + return false; + } + + /* For each length, check that the remaining bytes start with the 0x80 bit + * set and not the 0x40 bit. Then check for an overlong sequence, and for the + * excluded range 0xd800 to 0xdfff. */ + switch (ab) { + case 1: + /* 2-byte character. No further bytes to check for 0x80. Check first byte + * for for xx00 000x (overlong sequence). */ + if ((c & 0x3e) == 0) { + return false; + } + break; + + case 2: + /* 3-byte character. Check third byte for 0x80. Then check first 2 bytes for + * 1110 0000, xx0x xxxx (overlong sequence) or 1110 1101, 1010 xxxx (0xd800-0xdfff) */ + if ((*(++p) & 0xc0) != 0x80 || (c == 0xe0 && (d & 0x20) == 0) || (c == 0xed && d >= 0xa0)) { + return false; + } + break; + + case 3: + /* 4-byte character. Check 3rd and 4th bytes for 0x80. Then check first 2 + * bytes for for 1111 0000, xx00 xxxx (overlong sequence), then check for a + * character greater than 0x0010ffff (f4 8f bf bf) */ + if ((*(++p) & 0xc0) != 0x80 || (*(++p) & 0xc0) != 0x80 || (c == 0xf0 && (d & 0x30) == 0) || (c > 0xf4 || (c == 0xf4 && d > 0x8f))) { + return false; + } + break; + + EMPTY_SWITCH_DEFAULT_CASE(); + } + } + + return true; + +} + +#ifdef __aarch64__ +/* Adopted from range algorithm: https://github.com/cyb70289/utf8/ + * Thanks to @cyb70289 and @easyaspi314 + * + * MIT License + * + * Copyright (c) 2019 Yibo Cai + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#include + +/* + * Map high nibble of "First Byte" to legal character length minus 1 + * 0x00 ~ 0xBF --> 0 + * 0xC0 ~ 0xDF --> 1 + * 0xE0 ~ 0xEF --> 2 + * 0xF0 ~ 0xFF --> 3 + */ +static const uint8_t _first_len_tbl[] = { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 2, 3, +}; + +/* Map "First Byte" to 8-th item of range table (0xC2 ~ 0xF4) */ +static const uint8_t _first_range_tbl[] = { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 8, 8, 8, +}; + +/* + * Range table, map range index to min and max values + * Index 0 : 00 ~ 7F (First Byte, ascii) + * Index 1,2,3: 80 ~ BF (Second, Third, Fourth Byte) + * Index 4 : A0 ~ BF (Second Byte after E0) + * Index 5 : 80 ~ 9F (Second Byte after ED) + * Index 6 : 90 ~ BF (Second Byte after F0) + * Index 7 : 80 ~ 8F (Second Byte after F4) + * Index 8 : C2 ~ F4 (First Byte, non ascii) + * Index 9~15 : illegal: u >= 255 && u <= 0 + */ +static const uint8_t _range_min_tbl[] = { + 0x00, 0x80, 0x80, 0x80, 0xA0, 0x80, 0x90, 0x80, + 0xC2, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, +}; +static const uint8_t _range_max_tbl[] = { + 0x7F, 0xBF, 0xBF, 0xBF, 0xBF, 0x9F, 0xBF, 0x8F, + 0xF4, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +}; + +/* + * This table is for fast handling four special First Bytes(E0,ED,F0,F4), after + * which the Second Byte are not 80~BF. It contains "range index adjustment". + * - The idea is to minus byte with E0, use the result(0~31) as the index to + * lookup the "range index adjustment". Then add the adjustment to original + * range index to get the correct range. + * - Range index adjustment + * +------------+---------------+------------------+----------------+ + * | First Byte | original range| range adjustment | adjusted range | + * +------------+---------------+------------------+----------------+ + * | E0 | 2 | 2 | 4 | + * +------------+---------------+------------------+----------------+ + * | ED | 2 | 3 | 5 | + * +------------+---------------+------------------+----------------+ + * | F0 | 3 | 3 | 6 | + * +------------+---------------+------------------+----------------+ + * | F4 | 4 | 4 | 8 | + * +------------+---------------+------------------+----------------+ + * - Below is a uint8x16x2 table, data is interleaved in NEON register. So I'm + * putting it vertically. 1st column is for E0~EF, 2nd column for F0~FF. + */ +static const uint8_t _range_adjust_tbl[] = { + /* index -> 0~15 16~31 <- index */ + /* E0 -> */ 2, 3, /* <- F0 */ + 0, 0, + 0, 0, + 0, 0, + 0, 4, /* <- F4 */ + 0, 0, + 0, 0, + 0, 0, + 0, 0, + 0, 0, + 0, 0, + 0, 0, + 0, 0, + /* ED -> */ 3, 0, + 0, 0, + 0, 0, +}; + +bool utf8_range(const unsigned char *data, size_t len) +{ + if (len >= 16) { + uint8x16_t prev_input = vdupq_n_u8(0); + uint8x16_t prev_first_len = vdupq_n_u8(0); + + /* Cached tables */ + const uint8x16_t first_len_tbl = vld1q_u8(_first_len_tbl); + const uint8x16_t first_range_tbl = vld1q_u8(_first_range_tbl); + const uint8x16_t range_min_tbl = vld1q_u8(_range_min_tbl); + const uint8x16_t range_max_tbl = vld1q_u8(_range_max_tbl); + const uint8x16x2_t range_adjust_tbl = vld2q_u8(_range_adjust_tbl); + + /* Cached values */ + const uint8x16_t const_1 = vdupq_n_u8(1); + const uint8x16_t const_2 = vdupq_n_u8(2); + const uint8x16_t const_e0 = vdupq_n_u8(0xE0); + const uint8x16_t const_7f = vdupq_n_u8(0x7F); + + /* We use two error registers to remove a dependency. */ + uint8x16_t error1 = vdupq_n_u8(0); + uint8x16_t error2 = vdupq_n_u8(0); + + while (len >= 64) { + const uint8x16_t input_1 = vld1q_u8(data); + const uint8x16_t input_2 = vld1q_u8(data + 16); + const uint8x16_t input_3 = vld1q_u8(data + 32); + const uint8x16_t input_4 = vld1q_u8(data + 48); + + uint64_t ascii_paired = vgetq_lane_u64(vreinterpretq_u64_u8(prev_first_len), 0); + if (ascii_paired == 0) { + uint8x16_t is_ascii_0 = vorrq_u8(input_1, input_2); + is_ascii_0 = vorrq_u8(is_ascii_0, input_3); + is_ascii_0 = vorrq_u8(is_ascii_0, input_4); + + uint8x16_t is_ascii = vqsubq_u8(is_ascii_0, const_7f); + uint64_t is_ascii_paired = vgetq_lane_u64(vreinterpretq_u64_u8(is_ascii), 0); + + /* ascii */ + if (is_ascii_paired == 0) { + const uint8x16_t high_nibbles_4 = vshrq_n_u8(input_4, 4); + const uint8x16_t first_len_4 = vqtbl1q_u8(first_len_tbl, high_nibbles_4); + + prev_input = input_4; + prev_first_len = first_len_4; + + data += 64; + len -= 64; + continue; + } + } + + /* high_nibbles = input >> 4 */ + const uint8x16_t high_nibbles_1 = vshrq_n_u8(input_1, 4); + const uint8x16_t high_nibbles_2 = vshrq_n_u8(input_2, 4); + const uint8x16_t high_nibbles_3 = vshrq_n_u8(input_3, 4); + const uint8x16_t high_nibbles_4 = vshrq_n_u8(input_4, 4); + + /* first_len = legal character length minus 1 */ + /* 0 for 00~7F, 1 for C0~DF, 2 for E0~EF, 3 for F0~FF */ + /* first_len = first_len_tbl[high_nibbles] */ + const uint8x16_t first_len_1 = vqtbl1q_u8(first_len_tbl, high_nibbles_1); + const uint8x16_t first_len_2 = vqtbl1q_u8(first_len_tbl, high_nibbles_2); + const uint8x16_t first_len_3 = vqtbl1q_u8(first_len_tbl, high_nibbles_3); + const uint8x16_t first_len_4 = vqtbl1q_u8(first_len_tbl, high_nibbles_4); + + /* First Byte: set range index to 8 for bytes within 0xC0 ~ 0xFF */ + /* range = first_range_tbl[high_nibbles] */ + uint8x16_t range_1 = vqtbl1q_u8(first_range_tbl, high_nibbles_1); + uint8x16_t range_2 = vqtbl1q_u8(first_range_tbl, high_nibbles_2); + uint8x16_t range_3 = vqtbl1q_u8(first_range_tbl, high_nibbles_3); + uint8x16_t range_4 = vqtbl1q_u8(first_range_tbl, high_nibbles_4); + + /* Second Byte: set range index to first_len */ + /* 0 for 00~7F, 1 for C0~DF, 2 for E0~EF, 3 for F0~FF */ + /* range |= (first_len, prev_first_len) << 1 byte */ + range_1 = vorrq_u8(range_1, vextq_u8(prev_first_len, first_len_1, 15)); + range_2 = vorrq_u8(range_2, vextq_u8(first_len_1, first_len_2, 15)); + range_3 = vorrq_u8(range_3, vextq_u8(first_len_2, first_len_3, 15)); + range_4 = vorrq_u8(range_4, vextq_u8(first_len_3, first_len_4, 15)); + + /* Third Byte: set range index to saturate_sub(first_len, 1) */ + /* 0 for 00~7F, 0 for C0~DF, 1 for E0~EF, 2 for F0~FF */ + uint8x16_t tmp1_1, tmp1_2, tmp1_3, tmp1_4; + /* tmp1 = (first_len, prev_first_len) << 2 bytes */ + tmp1_1 = vextq_u8(prev_first_len, first_len_1, 14); + tmp1_2 = vextq_u8(first_len_1, first_len_2, 14); + tmp1_3 = vextq_u8(first_len_2, first_len_3, 14); + tmp1_4 = vextq_u8(first_len_3, first_len_4, 14); + /* tmp1 = saturate_sub(tmp1, 1) */ + tmp1_1 = vqsubq_u8(tmp1_1, const_1); + tmp1_2 = vqsubq_u8(tmp1_2, const_1); + tmp1_3 = vqsubq_u8(tmp1_3, const_1); + tmp1_4 = vqsubq_u8(tmp1_4, const_1); + /* range |= tmp1 */ + range_1 = vorrq_u8(range_1, tmp1_1); + range_2 = vorrq_u8(range_2, tmp1_2); + range_3 = vorrq_u8(range_3, tmp1_3); + range_4 = vorrq_u8(range_4, tmp1_4); + + uint8x16_t tmp2_1, tmp2_2, tmp2_3, tmp2_4; + /* Fourth Byte: set range index to saturate_sub(first_len, 2) */ + /* 0 for 00~7F, 0 for C0~DF, 0 for E0~EF, 1 for F0~FF */ + /* tmp2 = (first_len, prev_first_len) << 3 bytes */ + tmp2_1 = vextq_u8(prev_first_len, first_len_1, 13); + tmp2_2 = vextq_u8(first_len_1, first_len_2, 13); + tmp2_3 = vextq_u8(first_len_2, first_len_3, 13); + tmp2_4 = vextq_u8(first_len_3, first_len_4, 13); + /* tmp2 = saturate_sub(tmp2, 2) */ + tmp2_1 = vqsubq_u8(tmp2_1, const_2); + tmp2_2 = vqsubq_u8(tmp2_2, const_2); + tmp2_3 = vqsubq_u8(tmp2_3, const_2); + tmp2_4 = vqsubq_u8(tmp2_4, const_2); + /* range |= tmp2 */ + range_1 = vorrq_u8(range_1, tmp2_1); + range_2 = vorrq_u8(range_2, tmp2_2); + range_3 = vorrq_u8(range_3, tmp2_3); + range_4 = vorrq_u8(range_4, tmp2_4); + + /* + * Now we have below range indices caluclated + * Correct cases: + * - 8 for C0~FF + * - 3 for 1st byte after F0~FF + * - 2 for 1st byte after E0~EF or 2nd byte after F0~FF + * - 1 for 1st byte after C0~DF or 2nd byte after E0~EF or + * 3rd byte after F0~FF + * - 0 for others + * Error cases: + * 9,10,11 if non ascii First Byte overlaps + * E.g., F1 80 C2 90 --> 8 3 10 2, where 10 indicates error + */ + + /* Adjust Second Byte range for special First Bytes(E0,ED,F0,F4) */ + /* See _range_adjust_tbl[] definition for details */ + /* Overlaps lead to index 9~15, which are illegal in range table */ + uint8x16_t shift1_1 = vextq_u8(prev_input, input_1, 15); + uint8x16_t shift1_2 = vextq_u8(input_1, input_2, 15); + uint8x16_t shift1_3 = vextq_u8(input_2, input_3, 15); + uint8x16_t shift1_4 = vextq_u8(input_3, input_4, 15); + uint8x16_t pos_1 = vsubq_u8(shift1_1, const_e0); + uint8x16_t pos_2 = vsubq_u8(shift1_2, const_e0); + uint8x16_t pos_3 = vsubq_u8(shift1_3, const_e0); + uint8x16_t pos_4 = vsubq_u8(shift1_4, const_e0); + range_1 = vaddq_u8(range_1, vqtbl2q_u8(range_adjust_tbl, pos_1)); + range_2 = vaddq_u8(range_2, vqtbl2q_u8(range_adjust_tbl, pos_2)); + range_3 = vaddq_u8(range_3, vqtbl2q_u8(range_adjust_tbl, pos_3)); + range_4 = vaddq_u8(range_4, vqtbl2q_u8(range_adjust_tbl, pos_4)); + + /* Load min and max values per calculated range index */ + uint8x16_t minv_1 = vqtbl1q_u8(range_min_tbl, range_1); + uint8x16_t minv_2 = vqtbl1q_u8(range_min_tbl, range_2); + uint8x16_t minv_3 = vqtbl1q_u8(range_min_tbl, range_3); + uint8x16_t minv_4 = vqtbl1q_u8(range_min_tbl, range_4); + uint8x16_t maxv_1 = vqtbl1q_u8(range_max_tbl, range_1); + uint8x16_t maxv_2 = vqtbl1q_u8(range_max_tbl, range_2); + uint8x16_t maxv_3 = vqtbl1q_u8(range_max_tbl, range_3); + uint8x16_t maxv_4 = vqtbl1q_u8(range_max_tbl, range_4); + + /* Check value range */ + error1 = vorrq_u8(error1, vcltq_u8(input_1, minv_1)); + error2 = vorrq_u8(error2, vcgtq_u8(input_1, maxv_1)); + error1 = vorrq_u8(error1, vcltq_u8(input_2, minv_2)); + error2 = vorrq_u8(error2, vcgtq_u8(input_2, maxv_2)); + error1 = vorrq_u8(error1, vcltq_u8(input_3, minv_3)); + error2 = vorrq_u8(error2, vcgtq_u8(input_3, maxv_3)); + error1 = vorrq_u8(error1, vcltq_u8(input_4, minv_4)); + error2 = vorrq_u8(error2, vcgtq_u8(input_4, maxv_4)); + + /* Merge the error vectors */ + uint8x16_t error = vorrq_u8(error1, error2); + + /* + * Take the max of each adjacent element, selecting the errors (0xFF) into + * the low 8 elements of the vector. The upper bits are ignored. + */ + uint8x16_t error_paired = vpmaxq_u8(error, error); + /* Extract the raw bit pattern of the low 8 elements. */ + uint64_t error_raw = vgetq_lane_u64(vreinterpretq_u64_u8(error_paired), 0); + /* If any bits are nonzero, there is an error. */ + if (error_raw != 0) { + return false; + } + + prev_input = input_4; + prev_first_len = first_len_4; + + data += 64; + len -= 64; + } + + while (len >= 16) { + const uint8x16_t input = vld1q_u8(data); + + /* high_nibbles = input >> 4 */ + const uint8x16_t high_nibbles = vshrq_n_u8(input, 4); + + /* first_len = legal character length minus 1 */ + /* 0 for 00~7F, 1 for C0~DF, 2 for E0~EF, 3 for F0~FF */ + /* first_len = first_len_tbl[high_nibbles] */ + const uint8x16_t first_len = vqtbl1q_u8(first_len_tbl, high_nibbles); + + /* First Byte: set range index to 8 for bytes within 0xC0 ~ 0xFF */ + /* range = first_range_tbl[high_nibbles] */ + uint8x16_t range = vqtbl1q_u8(first_range_tbl, high_nibbles); + + /* Second Byte: set range index to first_len */ + /* 0 for 00~7F, 1 for C0~DF, 2 for E0~EF, 3 for F0~FF */ + /* range |= (first_len, prev_first_len) << 1 byte */ + range = vorrq_u8(range, vextq_u8(prev_first_len, first_len, 15)); + + /* Third Byte: set range index to saturate_sub(first_len, 1) */ + /* 0 for 00~7F, 0 for C0~DF, 1 for E0~EF, 2 for F0~FF */ + uint8x16_t tmp1, tmp2; + /* tmp1 = (first_len, prev_first_len) << 2 bytes */ + tmp1 = vextq_u8(prev_first_len, first_len, 14); + /* tmp1 = saturate_sub(tmp1, 1) */ + tmp1 = vqsubq_u8(tmp1, const_1); + /* range |= tmp1 */ + range = vorrq_u8(range, tmp1); + + /* Fourth Byte: set range index to saturate_sub(first_len, 2) */ + /* 0 for 00~7F, 0 for C0~DF, 0 for E0~EF, 1 for F0~FF */ + /* tmp2 = (first_len, prev_first_len) << 3 bytes */ + tmp2 = vextq_u8(prev_first_len, first_len, 13); + /* tmp2 = saturate_sub(tmp2, 2) */ + tmp2 = vqsubq_u8(tmp2, const_2); + /* range |= tmp2 */ + range = vorrq_u8(range, tmp2); + + /* + * Now we have below range indices caluclated + * Correct cases: + * - 8 for C0~FF + * - 3 for 1st byte after F0~FF + * - 2 for 1st byte after E0~EF or 2nd byte after F0~FF + * - 1 for 1st byte after C0~DF or 2nd byte after E0~EF or + * 3rd byte after F0~FF + * - 0 for others + * Error cases: + * 9,10,11 if non ascii First Byte overlaps + * E.g., F1 80 C2 90 --> 8 3 10 2, where 10 indicates error + */ + + /* Adjust Second Byte range for special First Bytes(E0,ED,F0,F4) */ + /* See _range_adjust_tbl[] definition for details */ + /* Overlaps lead to index 9~15, which are illegal in range table */ + uint8x16_t shift1 = vextq_u8(prev_input, input, 15); + uint8x16_t pos = vsubq_u8(shift1, const_e0); + range = vaddq_u8(range, vqtbl2q_u8(range_adjust_tbl, pos)); + + /* Load min and max values per calculated range index */ + uint8x16_t minv = vqtbl1q_u8(range_min_tbl, range); + uint8x16_t maxv = vqtbl1q_u8(range_max_tbl, range); + + /* Check value range */ + error1 = vorrq_u8(error1, vcltq_u8(input, minv)); + error2 = vorrq_u8(error2, vcgtq_u8(input, maxv)); + + /* Merge the error vectors */ + uint8x16_t error = vorrq_u8(error1, error2); + + /* + * Take the max of each adjacent element, selecting the errors (0xFF) into + * the low 8 elements of the vector. The upper bits are ignored. + */ + uint8x16_t error_paired = vpmaxq_u8(error, error); + /* Extract the raw bit pattern of the low 8 elements. */ + uint64_t error_raw = vgetq_lane_u64(vreinterpretq_u64_u8(error_paired), 0); + /* If any bits are nonzero, there is an error. */ + if (error_raw != 0) { + return false; + } + + prev_input = input; + prev_first_len = first_len; + + data += 16; + len -= 16; + } + + /* Merge our error counters together */ + error1 = vorrq_u8(error1, error2); + + uint64_t error_raw_last = vgetq_lane_u64(vreinterpretq_u64_u8(error1), 0); + /* Delay error check till loop ends */ + if (error_raw_last != 0) { + return false; + } + + /* Find previous token (not 80~BF) */ + uint32_t token4; + vst1q_lane_u32(&token4, vreinterpretq_u32_u8(prev_input), 3); + + const int8_t *token = (const int8_t *)&token4; + int lookahead = 0; + if (token[3] > (int8_t)0xBF) { + lookahead = 1; + } else if (token[2] > (int8_t)0xBF) { + lookahead = 2; + } else if (token[1] > (int8_t)0xBF) { + lookahead = 3; + } + + data -= lookahead; + len += lookahead; + } + + /* Check remaining bytes with naive method */ + return utf8_naive(data, len); +} +#endif + /* SSE2-based function for validating UTF-8 strings * A faster implementation which uses AVX2 instructions follows */ static bool mb_fast_check_utf8_default(zend_string *str) @@ -4780,82 +5278,13 @@ static bool mb_fast_check_utf8_default(zend_string *str) return true; # else - /* This UTF-8 validation function is derived from PCRE2 */ size_t length = ZSTR_LEN(str); - /* Table of the number of extra bytes, indexed by the first byte masked with - 0x3f. The highest number for a valid UTF-8 first byte is in fact 0x3d. */ - static const uint8_t utf8_table[] = { - 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, - 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, - 2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2, - 3,3,3,3,3,3,3,3 - }; - - for (; length > 0; p++) { - uint32_t d; - unsigned char c = *p; - length--; - - if (c < 128) { - /* ASCII character */ - continue; - } - - if (c < 0xc0) { - /* Isolated 10xx xxxx byte */ - return false; - } - - if (c >= 0xf5) { - return false; - } - - uint32_t ab = utf8_table[c & 0x3f]; /* Number of additional bytes (1-3) */ - if (length < ab) { - /* Missing bytes */ - return false; - } - length -= ab; - - /* Check top bits in the second byte */ - if (((d = *(++p)) & 0xc0) != 0x80) { - return false; - } - - /* For each length, check that the remaining bytes start with the 0x80 bit - * set and not the 0x40 bit. Then check for an overlong sequence, and for the - * excluded range 0xd800 to 0xdfff. */ - switch (ab) { - case 1: - /* 2-byte character. No further bytes to check for 0x80. Check first byte - * for for xx00 000x (overlong sequence). */ - if ((c & 0x3e) == 0) { - return false; - } - break; - - case 2: - /* 3-byte character. Check third byte for 0x80. Then check first 2 bytes for - * 1110 0000, xx0x xxxx (overlong sequence) or 1110 1101, 1010 xxxx (0xd800-0xdfff) */ - if ((*(++p) & 0xc0) != 0x80 || (c == 0xe0 && (d & 0x20) == 0) || (c == 0xed && d >= 0xa0)) { - return false; - } - break; - - case 3: - /* 4-byte character. Check 3rd and 4th bytes for 0x80. Then check first 2 - * bytes for for 1111 0000, xx00 xxxx (overlong sequence), then check for a - * character greater than 0x0010ffff (f4 8f bf bf) */ - if ((*(++p) & 0xc0) != 0x80 || (*(++p) & 0xc0) != 0x80 || (c == 0xf0 && (d & 0x30) == 0) || (c > 0xf4 || (c == 0xf4 && d > 0x8f))) { - return false; - } - break; - - EMPTY_SWITCH_DEFAULT_CASE(); - } - } - - return true; +# if defined(__aarch64__) + /* use to range algorithm, if less than 16 bytes, fallback to naive algorithm. */ + return utf8_range(p, length); +# else + return utf8_naive(p, length); +# endif /* if defined(__aarch64__) */ # endif } diff --git a/ext/mbstring/tests/utf_encodings.phpt b/ext/mbstring/tests/utf_encodings.phpt index 06c35b1546521..9253a6b57de51 100644 --- a/ext/mbstring/tests/utf_encodings.phpt +++ b/ext/mbstring/tests/utf_encodings.phpt @@ -820,7 +820,8 @@ $truncated16byte = [ "k\x08`\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xef\xbf", "k\x08`\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xf0", "k\x08`\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xf0\xbf", - "k\x08`\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xf0\xbf\xbf" + "k\x08`\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xf0\xbf\xbf", + "0123456789abcd\xe3\x810123456789abcdef" ]; foreach ($truncated16byte as $trunc) { if (mb_check_encoding($trunc, 'UTF-8'))