highlighter: Fix regions and patterns inside regions (zyedidia#2840)

* highlighter: Fix regions and patterns inside regions

* highlighting: Remove 2nd recursive highlightRegion() call

...and add limitGroup checks to pattern search.

* yaml: Add TODO type highlighting

* highlighting: Don't stop in highlightRegion() at empty lines

...because possible region line end pattern must be detected.

* syntax/sh: Correct string handling due to additional pattern handling

* syntax/sh: Remove slash in variables

* highlighter: Accept nested region only in case it's within the current reagion

* highlighter: Accept nested patterns only in case it's within the current reagion

* highlighter: Don't search for nesting in case the region end was found at start

Author: JoeKar

pkg/highlight/highlighter.go

@@ -134,51 +134,33 @@ func (h *Highlighter) highlightRegion(highlights LineMatch, start int, canMatchE
 		}
 	}
 
-	loc := findIndex(curRegion.end, curRegion.skip, line)
-	if loc != nil {
-		if !statesOnly {
-			highlights[start+loc[0]] = curRegion.limitGroup
-		}
-		if curRegion.parent == nil {
-			if !statesOnly {
-				highlights[start+loc[1]] = 0
-			}
-			h.highlightEmptyRegion(highlights, start+loc[1], canMatchEnd, lineNum, sliceStart(line, loc[1]), statesOnly)
-			return highlights
-		}
-		if !statesOnly {
-			highlights[start+loc[1]] = curRegion.parent.group
-			h.highlightRegion(highlights, start, false, lineNum, sliceEnd(line, loc[0]), curRegion, statesOnly)
-		}
-		h.highlightRegion(highlights, start+loc[1], canMatchEnd, lineNum, sliceStart(line, loc[1]), curRegion.parent, statesOnly)
-		return highlights
-	}
-
-	if lineLen == 0 {
-		if canMatchEnd {
-			h.lastRegion = curRegion
+	var firstRegion *region
+	firstLoc := []int{lineLen, 0}
+	searchNesting := true
+	endLoc := findIndex(curRegion.end, curRegion.skip, line)
+	if endLoc != nil {
+		if start == endLoc[0] {
+			searchNesting = false
+		} else {
+			firstLoc = endLoc
 		}
-
-		return highlights
 	}
-
-	firstLoc := []int{lineLen, 0}
-
-	var firstRegion *region
-	for _, r := range curRegion.rules.regions {
-		loc := findIndex(r.start, r.skip, line)
-		if loc != nil {
-			if loc[0] < firstLoc[0] {
-				firstLoc = loc
-				firstRegion = r
+	if searchNesting {
+		for _, r := range curRegion.rules.regions {
+			loc := findIndex(r.start, r.skip, line)
+			if loc != nil {
+				if loc[0] < firstLoc[0] {
+					firstLoc = loc
+					firstRegion = r
+				}
 			}
 		}
 	}
-	if firstLoc[0] != lineLen {
+	if firstRegion != nil && firstLoc[0] != lineLen {
 		if !statesOnly {
 			highlights[start+firstLoc[0]] = firstRegion.limitGroup
 		}
-		h.highlightRegion(highlights, start, false, lineNum, sliceEnd(line, firstLoc[0]), curRegion, statesOnly)
+		h.highlightEmptyRegion(highlights, start+firstLoc[1], canMatchEnd, lineNum, sliceStart(line, firstLoc[1]), statesOnly)
 		h.highlightRegion(highlights, start+firstLoc[1], canMatchEnd, lineNum, sliceStart(line, firstLoc[1]), firstRegion, statesOnly)
 		return highlights
 	}
@@ -189,11 +171,17 @@ func (h *Highlighter) highlightRegion(highlights LineMatch, start int, canMatchE
 			fullHighlights[i] = curRegion.group
 		}
 
-		for _, p := range curRegion.rules.patterns {
-			matches := findAllIndex(p.regex, line)
-			for _, m := range matches {
-				for i := m[0]; i < m[1]; i++ {
-					fullHighlights[i] = p.group
+		if searchNesting {
+			for _, p := range curRegion.rules.patterns {
+				if curRegion.group == curRegion.limitGroup || p.group == curRegion.limitGroup {
+					matches := findAllIndex(p.regex, line)
+					for _, m := range matches {
+						if ((endLoc == nil) || (m[0] < endLoc[0])) {
+							for i := m[0]; i < m[1]; i++ {
+								fullHighlights[i] = p.group
+							}
+						}
+					}
 				}
 			}
 		}
@@ -204,6 +192,25 @@ func (h *Highlighter) highlightRegion(highlights LineMatch, start int, canMatchE
 		}
 	}
 
+	loc := endLoc
+	if loc != nil {
+		if !statesOnly {
+			highlights[start+loc[0]] = curRegion.limitGroup
+		}
+		if curRegion.parent == nil {
+			if !statesOnly {
+				highlights[start+loc[1]] = 0
+			}
+			h.highlightEmptyRegion(highlights, start+loc[1], canMatchEnd, lineNum, sliceStart(line, loc[1]), statesOnly)
+			return highlights
+		}
+		if !statesOnly {
+			highlights[start+loc[1]] = curRegion.parent.group
+		}
+		h.highlightRegion(highlights, start+loc[1], canMatchEnd, lineNum, sliceStart(line, loc[1]), curRegion.parent, statesOnly)
+		return highlights
+	}
+
 	if canMatchEnd {
 		h.lastRegion = curRegion
 	}
@@ -220,8 +227,8 @@ func (h *Highlighter) highlightEmptyRegion(highlights LineMatch, start int, canM
 		return highlights
 	}
 
-	firstLoc := []int{lineLen, 0}
 	var firstRegion *region
+	firstLoc := []int{lineLen, 0}
 	for _, r := range h.Def.rules.regions {
 		loc := findIndex(r.start, r.skip, line)
 		if loc != nil {
@@ -231,7 +238,7 @@ func (h *Highlighter) highlightEmptyRegion(highlights LineMatch, start int, canM
 			}
 		}
 	}
-	if firstLoc[0] != lineLen {
+	if firstRegion != nil && firstLoc[0] != lineLen {
 		if !statesOnly {
 			highlights[start+firstLoc[0]] = firstRegion.limitGroup
 		}

runtime/syntax/sh.yaml

@@ -42,18 +42,18 @@ rules:
     - statement: " (-[A-Za-z]+|--[a-z]+)"
 
     - identifier: "\\$\\{[0-9A-Za-z_:!%&=+#~@*^$?, .\\-\\/\\[\\]]+\\}"
-    - identifier: "\\$[0-9A-Za-z_:!%&=+#~@*^$?,\\-\\/\\[\\]]+"
+    - identifier: "\\$[0-9A-Za-z_:!%&=+#~@*^$?,\\-\\[\\]]+"
 
     - constant.string:
         start: "\""
         end: "\""
         skip: "\\\\."
-        rules:
-            - constant.specialChar: "\\\\."
+        rules: []
 
     - constant.string:
         start: "'"
         end: "'"
+        skip: "\\\\."
         rules: []
 
     - comment:

runtime/syntax/yaml.yaml

@@ -30,6 +30,5 @@ rules:
     - comment:
         start: "#"
         end: "$"
-        rules: []
-
-
+        rules:
+            - todo: "(TODO|XXX|FIXME):?"