Merge pull request #51 from ruby-syntax-tree/pattern-matching

Pattern matching

Author: kddnewton

Gemfile

@@ -3,8 +3,3 @@
 source "https://rubygems.org"
 
 gemspec
-
-gem "benchmark-ips"
-gem "parser"
-gem "ruby_parser"
-gem "stackprof"

Gemfile.lock

@@ -6,23 +6,15 @@ PATH
 GEM
   remote: https://rubygems.org/
   specs:
-    ast (2.4.2)
-    benchmark-ips (2.10.0)
     docile (1.4.0)
     minitest (5.15.0)
-    parser (3.1.2.0)
-      ast (~> 2.4.1)
     rake (13.0.6)
-    ruby_parser (3.19.1)
-      sexp_processor (~> 4.16)
-    sexp_processor (4.16.0)
     simplecov (0.21.2)
       docile (~> 1.1)
       simplecov-html (~> 0.11)
       simplecov_json_formatter (~> 0.1)
     simplecov-html (0.12.3)
     simplecov_json_formatter (0.1.3)
-    stackprof (0.2.19)
 
 PLATFORMS
   arm64-darwin-21
@@ -32,14 +24,10 @@ PLATFORMS
   x86_64-linux
 
 DEPENDENCIES
-  benchmark-ips
   bundler
   minitest
-  parser
   rake
-  ruby_parser
   simplecov
-  stackprof
   syntax_tree!
 
 BUNDLED WITH

bin/bench

@@ -1,12 +1,17 @@
 #!/usr/bin/env ruby
 # frozen_string_literal: true
 
-require "bundler/setup"
-require "benchmark/ips"
+require "bundler/inline"
 
-require_relative "../lib/syntax_tree"
-require "ruby_parser"
-require "parser/current"
+gemfile do
+  source "https://rubygems.org"
+  gem "benchmark-ips"
+  gem "parser", require: "parser/current"
+  gem "ruby_parser"
+end
+
+$:.unshift(File.expand_path("../lib", __dir__))
+require "syntax_tree"
 
 def compare(filepath)
   prefix = "#{File.expand_path("..", __dir__)}/"
@@ -30,7 +35,7 @@ filepaths = ARGV
 if filepaths.empty?
   filepaths = [
     File.expand_path("bench", __dir__),
-    File.expand_path("../lib/syntax_tree.rb", __dir__)
+    File.expand_path("../lib/syntax_tree/node.rb", __dir__)
   ]
 end
 

bin/profile

@@ -1,19 +1,28 @@
 #!/usr/bin/env ruby
 # frozen_string_literal: true
 
-require "bundler/setup"
-require "stackprof"
+require "bundler/inline"
 
-filepath = File.expand_path("../lib/syntax_tree", __dir__)
-require_relative filepath
+gemfile do
+  source "https://rubygems.org"
+  gem "stackprof"
+end
+
+$:.unshift(File.expand_path("../lib", __dir__))
+require "syntax_tree"
 
 GC.disable
 
 StackProf.run(mode: :cpu, out: "tmp/profile.dump", raw: true) do
-  SyntaxTree.format(File.read("#{filepath}.rb"))
+  filepath = File.expand_path("../lib/syntax_tree/node.rb", __dir__)
+  SyntaxTree.format(File.read(filepath))
 end
 
 GC.enable
 
-`bundle exec stackprof --d3-flamegraph tmp/profile.dump > tmp/flamegraph.html`
-puts "open tmp/flamegraph.html"
+File.open("tmp/flamegraph.html", "w") do |file|
+  report = Marshal.load(IO.binread("tmp/profile.dump"))
+  StackProf::Report.new(report).print_d3_flamegraph(file)
+end
+
+`open tmp/flamegraph.html`

lib/syntax_tree/node.rb

@@ -35,6 +35,21 @@ def to(other)
       )
     end
 
+    def deconstruct
+      [start_line, start_char, start_column, end_line, end_char, end_column]
+    end
+
+    def deconstruct_keys(keys)
+      {
+        start_line: start_line,
+        start_char: start_char,
+        start_column: start_column,
+        end_line: end_line,
+        end_char: end_char,
+        end_column: end_column
+      }
+    end
+
     def self.token(line:, char:, column:, size:)
       new(
         start_line: line,
@@ -4334,16 +4349,20 @@ class Heredoc < Node
     # [String] the ending of the heredoc
     attr_reader :ending
 
+    # [Integer] how far to dedent the heredoc
+    attr_reader :dedent
+
     # [Array[ StringEmbExpr | StringDVar | TStringContent ]] the parts of the
     # heredoc string literal
     attr_reader :parts
 
     # [Array[ Comment | EmbDoc ]] the comments attached to this node
     attr_reader :comments
 
-    def initialize(beginning:, ending: nil, parts: [], location:, comments: [])
+    def initialize(beginning:, ending: nil, dedent: 0, parts: [], location:, comments: [])
       @beginning = beginning
       @ending = ending
+      @dedent = dedent
       @parts = parts
       @location = location
       @comments = comments
@@ -4538,7 +4557,12 @@ def format(q)
       parts << KeywordRestFormatter.new(keyword_rest) if keyword_rest
 
       contents = -> do
-        q.seplist(parts) { |part| q.format(part, stackable: false) }
+        q.group { q.seplist(parts) { |part| q.format(part, stackable: false) } }
+
+        # If there isn't a constant, and there's a blank keyword_rest, then we
+        # have an plain ** that needs to have a `then` after it in order to
+        # parse correctly on the next parse.
+        q.text(" then") if !constant && keyword_rest && keyword_rest.value.nil?
       end
 
       if constant
@@ -5594,11 +5618,17 @@ class MLHSParen < Node
     # [MLHS | MLHSParen] the contents inside of the parentheses
     attr_reader :contents
 
+    # [boolean] whether or not there is a trailing comma at the end of this
+    # list, which impacts destructuring. It's an attr_accessor so that while
+    # the syntax tree is being built it can be set by its parent node
+    attr_accessor :comma
+
     # [Array[ Comment | EmbDoc ]] the comments attached to this node
     attr_reader :comments
 
-    def initialize(contents:, location:, comments: [])
+    def initialize(contents:, comma: false, location:, comments: [])
       @contents = contents
+      @comma = comma
       @location = location
       @comments = comments
     end
@@ -5622,13 +5652,15 @@ def format(q)
 
       if parent.is_a?(MAssign) || parent.is_a?(MLHSParen)
         q.format(contents)
+        q.text(",") if comma
       else
         q.group(0, "(", ")") do
           q.indent do
             q.breakable("")
             q.format(contents)
           end
 
+          q.text(",") if comma
           q.breakable("")
         end
       end

lib/syntax_tree/parser.rb

@@ -516,12 +516,46 @@ def on_array(contents)
     def on_aryptn(constant, requireds, rest, posts)
       parts = [constant, *requireds, rest, *posts].compact
 
+      # If there aren't any parts (no constant, no positional arguments), then
+      # we're matching an empty array. In this case, we're going to look for the
+      # left and right brackets explicitly. Otherwise, we'll just use the bounds
+      # of the various parts.
+      location =
+        if parts.empty?
+          find_token(LBracket).location.to(find_token(RBracket).location)
+        else
+          parts[0].location.to(parts[-1].location)
+        end
+
+      # If there's the optional then keyword, then we'll delete that and use it
+      # as the end bounds of the location.
+      if token = find_token(Kw, "then", consume: false)
+        tokens.delete(token)
+        location = location.to(token.location)
+      end
+
+      # If there is a plain *, then we're going to fix up the location of it
+      # here because it currently doesn't have anything to use for its precise
+      # location. If we hit a comma, then we've gone too far.
+      if rest.is_a?(VarField) && rest.value.nil?
+        tokens.rindex do |token|
+          case token
+          in Op[value: "*"]
+            rest = VarField.new(value: nil, location: token.location)
+            break
+          in Comma
+            break
+          else
+          end
+        end
+      end
+
       AryPtn.new(
         constant: constant,
         requireds: requireds || [],
         rest: rest,
         posts: posts || [],
-        location: parts[0].location.to(parts[-1].location)
+        location: location
       )
     end
 
@@ -1373,15 +1407,35 @@ def on_float(value)
     #     VarField right
     #   ) -> FndPtn
     def on_fndptn(constant, left, values, right)
-      beginning = constant || find_token(LBracket)
-      ending = find_token(RBracket)
+      # The opening of this find pattern is either going to be a left bracket, a
+      # right left parenthesis, or the left splat. We're going to use this to
+      # determine how to find the closing of the pattern, as well as determining
+      # the location of the node.
+      opening =
+        find_token(LBracket, consume: false) ||
+        find_token(LParen, consume: false) ||
+        left
+
+      # The closing is based on the opening, which is either the matched
+      # punctuation or the right splat.
+      closing =
+        case opening
+        in LBracket
+          tokens.delete(opening)
+          find_token(RBracket)
+        in LParen
+          tokens.delete(opening)
+          find_token(RParen)
+        else
+          right
+        end
 
       FndPtn.new(
         constant: constant,
         left: left,
         values: values,
         right: right,
-        location: beginning.location.to(ending.location)
+        location: (constant || opening).location.to(closing.location)
       )
     end
 
@@ -1468,6 +1522,7 @@ def on_heredoc_dedent(string, width)
       @heredocs[-1] = Heredoc.new(
         beginning: heredoc.beginning,
         ending: heredoc.ending,
+        dedent: width,
         parts: string.parts,
         location: heredoc.location
       )
@@ -1481,6 +1536,7 @@ def on_heredoc_end(value)
       @heredocs[-1] = Heredoc.new(
         beginning: heredoc.beginning,
         ending: value.chomp,
+        dedent: heredoc.dedent,
         parts: heredoc.parts,
         location:
           Location.new(
@@ -1501,12 +1557,23 @@ def on_heredoc_end(value)
     #     (nil | VarField) keyword_rest
     #   ) -> HshPtn
     def on_hshptn(constant, keywords, keyword_rest)
+      # Create an artificial VarField if we find an extra ** on the end
+      if !keyword_rest && (token = find_token(Op, "**", consume: false))
+        tokens.delete(token)
+        keyword_rest = VarField.new(value: nil, location: token.location)
+      end
+
+      # Delete the optional then keyword
+      if token = find_token(Kw, "then", consume: false)
+        tokens.delete(token)
+      end
+
       parts = [constant, *keywords&.flatten(1), keyword_rest].compact
       location =
-        if parts.empty?
-          find_token(LBrace).location.to(find_token(RBrace).location)
-        else
+        if parts.any?
           parts[0].location.to(parts[-1].location)
+        else
+          find_token(LBrace).location.to(find_token(RBrace).location)
         end
 
       HshPtn.new(
@@ -2638,6 +2705,7 @@ def on_string_literal(string)
         Heredoc.new(
           beginning: heredoc.beginning,
           ending: heredoc.ending,
+          dedent: heredoc.dedent,
           parts: string.parts,
           location: heredoc.location
         )
@@ -3190,6 +3258,7 @@ def on_xstring_literal(xstring)
         Heredoc.new(
           beginning: heredoc.beginning,
           ending: heredoc.ending,
+          dedent: heredoc.dedent,
           parts: xstring.parts,
           location: heredoc.location
         )