libBasic: implement extended grapheme cluster segmentation algorithm

This is only for the frontend, not for stdlib.  The implementation is very
slow, optimizing it is the next step.

rdar://16755123 rdar://16013860


Swift SVN r18928

Author: gribozavr

CMakeLists.txt

@@ -545,6 +545,8 @@ set(SWIFTLIB_SUBDIR ".")
 set(SWIFT_GYB_FLAGS
   "-DCMAKE_SIZEOF_VOID_P=${CMAKE_SIZEOF_VOID_P}"
   "-DCMAKE_SYSTEM_PROCESSOR=${CMAKE_SYSTEM_PROCESSOR}"
+  "-DunicodeGraphemeBreakPropertyFile=${SWIFT_SOURCE_DIR}/utils/UnicodeData/GraphemeBreakProperty.txt"
+  "-DunicodeGraphemeBreakTestFile=${SWIFT_SOURCE_DIR}/utils/UnicodeData/GraphemeBreakTest.txt"
   "--test" # Run gyb's self-tests whenever we use it.  They're cheap
            # enough and it keeps us honest.
   )

include/swift/Basic/Unicode.h

@@ -28,6 +28,53 @@ static inline bool isSingleExtendedGraphemeCluster(StringRef S) {
   return First == S;
 }
 
+enum class GraphemeClusterBreakProperty : uint8_t {
+  Other,
+  CR,
+  LF,
+  Control,
+  Extend,
+  Regional_Indicator,
+  Prepend,
+  SpacingMark,
+  L,
+  V,
+  T,
+  LV,
+  LVT,
+};
+
+/// Extended grapheme cluster boundary rules, represented as a matrix.  Indexed
+/// by first code point, then by second code point in least-significant-bit
+/// order.  A set bit means that a boundary is prohibited between two code
+/// points.
+extern uint16_t ExtendedGraphemeClusterNoBoundaryRulesMatrix[];
+
+/// Returns the value of the Grapheme_Cluster_Break property for a given code
+/// point.
+GraphemeClusterBreakProperty getGraphemeClusterBreakProperty(uint32_t C);
+
+/// Returns true if there is always an extended grapheme cluster boundary
+/// after a code point with a given property value.  Use only for optimization,
+/// to skip calculating Grapheme_Cluster_Break property for the second code
+/// point.
+static inline bool
+isExtendedGraphemeClusterBoundaryAfter(GraphemeClusterBreakProperty GCB1) {
+  auto RuleRow =
+      ExtendedGraphemeClusterNoBoundaryRulesMatrix[static_cast<unsigned>(GCB1)];
+  return RuleRow == 0;
+}
+
+/// Determine if there is an extended grapheme cluster boundary between code
+/// points with given Grapheme_Cluster_Break property values.
+static inline bool
+isExtendedGraphemeClusterBoundary(GraphemeClusterBreakProperty GCB1,
+                                  GraphemeClusterBreakProperty GCB2) {
+  auto RuleRow =
+      ExtendedGraphemeClusterNoBoundaryRulesMatrix[static_cast<unsigned>(GCB1)];
+  return !(RuleRow & (1 << static_cast<unsigned>(GCB2)));
+}
+
 } // namespace unicode
 } // namespace swift
 

lib/Basic/CMakeLists.txt

@@ -1,3 +1,7 @@
+set(UNICODE_TABLES UnicodeExtendedGraphemeClusters.cpp.gyb)
+
+handle_gyb_sources(UNICODE_TABLES)
+
 add_swift_library(swiftBasic
   Cache.cpp
   Demangle.cpp
@@ -21,7 +25,10 @@ add_swift_library(swiftBasic
   Unix/TaskQueue.inc
 
   # Platform-agnostic fallback TaskQueue implementation
-  Default/TaskQueue.inc)
+  Default/TaskQueue.inc
+
+  ${UNICODE_TABLES}
+  )
 
 set(SWIFT_VERSION "1.0")
 message(STATUS "Swift version: ${SWIFT_VERSION}")

lib/Basic/Unicode.cpp

@@ -16,13 +16,47 @@
 using namespace swift;
 
 StringRef swift::unicode::extractFirstExtendedGraphemeCluster(StringRef S) {
-  // FIXME: implement as described in Unicode Standard Annex #29.
+  // Extended grapheme cluster segmentation algorithm as described in Unicode
+  // Standard Annex #29.
   if (S.empty())
     return StringRef();
 
-  // FIXME: deal with broken code unit sequences.
-  // For now, just extract the first code point.
-  unsigned CodeUnitSeqLen = getNumBytesForUTF8(S[0]);
-  return S.slice(0, CodeUnitSeqLen);
+  const UTF8 *SourceStart = reinterpret_cast<const UTF8 *>(S.data());
+
+  const UTF8 *SourceNext = SourceStart;
+  UTF32 C[2];
+  UTF32 *TargetStart = C;
+
+  ConvertUTF8toUTF32(&SourceNext, SourceStart + S.size(), &TargetStart, C + 1,
+                     lenientConversion);
+  if (TargetStart == C) {
+    // The source string contains an ill-formed subsequence at the end.
+    return S;
+  }
+
+  GraphemeClusterBreakProperty GCBForC0 = getGraphemeClusterBreakProperty(C[0]);
+  while (true) {
+    if (isExtendedGraphemeClusterBoundaryAfter(GCBForC0))
+      return S.slice(0, SourceNext - SourceStart);
+
+    size_t C1Offset = SourceNext - SourceStart;
+    ConvertUTF8toUTF32(&SourceNext, SourceStart + S.size(), &TargetStart, C + 2,
+                       lenientConversion);
+
+    if (TargetStart == C + 1) {
+      // End of source string or the source string contains an ill-formed
+      // subsequence at the end.
+      return S.slice(0, C1Offset);
+    }
+
+    GraphemeClusterBreakProperty GCBForC1 =
+        getGraphemeClusterBreakProperty(C[1]);
+    if (isExtendedGraphemeClusterBoundary(GCBForC0, GCBForC1))
+      return S.slice(0, C1Offset);
+
+    C[0] = C[1];
+    TargetStart = C + 1;
+    GCBForC0 = GCBForC1;
+  }
 }
 

lib/Basic/UnicodeExtendedGraphemeClusters.cpp.gyb

@@ -0,0 +1,121 @@
+%# -*- mode: C++ -*-
+
+%# Ignore the following admonition; it applies to the resulting .cpp file only
+//// Automatically Generated From UnicodeExtendedGraphemeClusters.cpp.gyb.
+//// Do Not Edit Directly!
+//===----------------------------------------------------------------------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2015 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See http://swift.org/LICENSE.txt for license information
+// See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+%{
+
+import re
+
+# Grapheme_Cluster_Break property.  An array of tuples (startCodePoint,
+# endCodePoint, value).
+graphemeBreakProperty = []
+
+with open(unicodeGraphemeBreakPropertyFile, 'rb') as f:
+    for line in f:
+        # Strip comments.
+        line = re.sub('#.*', '', line)
+
+        # Single code point?
+        m = re.match('([0-9A-F]+) +; +([a-zA-Z]+) ', line)
+        if m:
+            codePoint = int(m.group(1), 16)
+            value = m.group(2)
+            graphemeBreakProperty += [(codePoint, codePoint, value)]
+            continue
+
+        # Range of code points?
+        m = re.match('([0-9A-F]+)..([0-9A-F]+) +; +([a-zA-Z_]+) ', line)
+        if m:
+            startCodePoint = int(m.group(1), 16)
+            endCodePoint = int(m.group(2), 16)
+            value = m.group(3)
+            graphemeBreakProperty += [(startCodePoint, endCodePoint, value)]
+
+}%
+
+#include "swift/Basic/Unicode.h"
+
+swift::unicode::GraphemeClusterBreakProperty
+swift::unicode::getGraphemeClusterBreakProperty(uint32_t C) {
+  // FIXME: replace linear search with a trie lookup.
+
+% for startCodePoint,endCodePoint,value in graphemeBreakProperty:
+%   if startCodePoint == 0:
+  if (C <= ${endCodePoint})
+%   else:
+  if (C >= ${startCodePoint} && C <= ${endCodePoint})
+%   end
+    return GraphemeClusterBreakProperty::${value};
+% end
+
+  return GraphemeClusterBreakProperty::Other;
+}
+
+%{
+
+# The order should be consistent with 'GraphemeClusterBreakProperty' enum.
+anyGraphemePropertyValue = [
+  'Other', 'CR', 'LF', 'Control', 'Extend', 'Regional_Indicator', 'Prepend',
+  'SpacingMark', 'L', 'V', 'T', 'LV', 'LVT',
+]
+
+# Rules to determine extended grapheme cluster boundaries, as defined in
+# 'Grapheme Break Chart', ucd/auxiliary/GraphemeBreakTest.html, Unicode 6.3.0.
+extendedGraphemeClusterRules = [
+  ( [ 'CR' ], 'no_boundary', [ 'LF' ] ),
+  ( [ 'Control', 'CR', 'LF' ], 'boundary', anyGraphemePropertyValue ),
+  ( anyGraphemePropertyValue, 'boundary', [ 'Control', 'CR', 'LF' ] ),
+  ( [ 'L' ], 'no_boundary', [ 'L', 'V', 'LV', 'LVT' ] ),
+  ( [ 'LV', 'V' ], 'no_boundary', [ 'V', 'T' ] ),
+  ( [ 'LVT', 'T' ], 'no_boundary', [ 'T' ] ),
+  ( [ 'Regional_Indicator' ], 'no_boundary', [ 'Regional_Indicator' ] ),
+  ( anyGraphemePropertyValue, 'no_boundary', [ 'Extend' ] ),
+  ( anyGraphemePropertyValue, 'no_boundary', [ 'SpacingMark' ] ),
+  ( [ 'Prepend' ], 'no_boundary', anyGraphemePropertyValue ),
+  ( anyGraphemePropertyValue, 'boundary', anyGraphemePropertyValue ),
+]
+
+# Expand the rules into a matrix.
+extendedGraphemeClusterRulesMatrix = {}
+for first in anyGraphemePropertyValue:
+    extendedGraphemeClusterRulesMatrix[first] = \
+        dict.fromkeys(anyGraphemePropertyValue, None)
+
+for firstList,action,secondList in reversed(extendedGraphemeClusterRules):
+    for first in firstList:
+        for second in secondList:
+            extendedGraphemeClusterRulesMatrix[first][second] = action
+
+# Make sure we can pack one row of the matrix into a 'uint16_t'.
+assert(len(anyGraphemePropertyValue) <= 16)
+
+}%
+
+uint16_t swift::unicode::ExtendedGraphemeClusterNoBoundaryRulesMatrix[] = {
+% for first in anyGraphemePropertyValue:
+%   # Retrieve a row that corresponds to this first code point.
+%   row = extendedGraphemeClusterRulesMatrix[first]
+
+%   # Change strings into bits.
+%   bits = [ row[second] == 'no_boundary' for second in anyGraphemePropertyValue ]
+
+%   # Pack bits into an integer.
+%   packed = sum([ bits[i] * pow(2, i) for i in range(0, len(bits)) ])
+
+  ${packed},
+% end
+};
+

test/stdlib/CharacterTypes.swift

@@ -21,9 +21,7 @@ func testTypeInference() {
   isCharacter(&ch2)
   var ch4: Character = "例"
   isCharacter(&ch3)
-  // FIXME: this should pass when we implement grapheme cluster extraction
-  // correctly.
-  var ch5: Character = "\u304b\u3099" // expected-error {{cannot convert the expression's type 'String' to type 'Character'}}
+  var ch5: Character = "\u304b\u3099"
   isCharacter(&ch4)
 
   var s1 = ""

unittests/Basic/CMakeLists.txt

@@ -1,9 +1,15 @@
+set(GENERATED_TESTS UnicodeGraphemeBreakTest.cpp.gyb)
+
+handle_gyb_sources(GENERATED_TESTS)
+
 add_swift_unittest(SwiftBasicTests
   SourceManager.cpp
   TreeScopedHashTableTests.cpp
   StringExtrasTest.cpp
   SuccessorMapTest.cpp
   Unicode.cpp
+
+  ${GENERATED_TESTS}
   )
 
 target_link_libraries(SwiftBasicTests